def load_ptb_train(path): """ Deprecated, moved to neon.data.dataloaders. """ logger.error('This function has moved, import from neon.data.dataloaders') from neon.data.dataloaders import load_ptb_train # noqa return load_ptb_train(path)
default='lstm', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 20 time_steps = 20 hidden_size = 200 gradient_clip_norm = 5 # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) # download penn treebank train_path = load_ptb_train(path=args.data_dir) valid_path = load_ptb_test(path=args.data_dir) # define a custom function to parse the input into individual tokens, which for # this data, splits into individual words. This can be passed into the Text # object during dataset creation as seen below. def tokenizer(s): return s.replace('\n', '<eos>').split() # load data and parse on word-level train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False)
parser = NeonArgparser(__doc__) parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 20 time_steps = 20 hidden_size = 200 gradient_clip_norm = 5 # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) # download penn treebank train_path = load_ptb_train(path=args.data_dir) valid_path = load_ptb_test(path=args.data_dir) # define a custom function to parse the input into individual tokens, which for # this data, splits into individual words. This can be passed into the Text # object during dataset creation as seen below. def tokenizer(s): return s.replace('\n', '<eos>').split() # load data and parse on word-level train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False) valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=tokenizer, onehot_input=False) # weight initialization
def load_ptb_train(path): logger.error('This function has moved, import from neon.data.dataloaders') from neon.data.dataloaders import load_ptb_train # noqa return load_ptb_train(path)