def load_ptb_train(path):
"""
Deprecated, moved to neon.data.dataloaders.
"""
logger.error('This function has moved, import from neon.data.dataloaders')
from neon.data.dataloaders import load_ptb_train # noqa
return load_ptb_train(path)
default='lstm',
choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_ptb_train(path=args.data_dir)
valid_path = load_ptb_test(path=args.data_dir)
# define a custom function to parse the input into individual tokens, which for
# this data, splits into individual words. This can be passed into the Text
# object during dataset creation as seen below.
def tokenizer(s):
return s.replace('\n', '<eos>').split()
# load data and parse on word-level
train_set = Text(time_steps,
train_path,
tokenizer=tokenizer,
onehot_input=False)
parser = NeonArgparser(__doc__)
parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_ptb_train(path=args.data_dir)
valid_path = load_ptb_test(path=args.data_dir)
# define a custom function to parse the input into individual tokens, which for
# this data, splits into individual words. This can be passed into the Text
# object during dataset creation as seen below.
def tokenizer(s):
return s.replace('\n', '<eos>').split()
# load data and parse on word-level
train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=tokenizer,
onehot_input=False)
# weight initialization
def load_ptb_train(path):
logger.error('This function has moved, import from neon.data.dataloaders')
from neon.data.dataloaders import load_ptb_train # noqa
return load_ptb_train(path)
