else:
train, test, valid = load_from_lines(args.source_path,
args.batch_size,
min_freq=args.min_freq,
max_size=args.max_size,
gpu=args.gpu,
dev=args.dev,
test=args.text)
# save
if args.cache_data:
train.to_disk('data/%s_train.pt' % prefix)
test.to_disk('data/%s_test.pt' % prefix)
valid.to_disk('data/%s_valid.pt' % prefix)
# load from file
else:
train = PairedDataset.from_disk('data/%s_train.pt' % prefix)
test = PairedDataset.from_disk('data/%s_test.pt' % prefix)
valid = PairedDataset.from_disk('data/%s_valid.pt' % prefix)
train.set_gpu(args.gpu)
test.set_gpu(args.gpu)
valid.set_gpu(args.gpu)
train.set_batch_size(args.batch_size)
test.set_batch_size(args.batch_size)
valid.set_batch_size(args.batch_size)
print("* Number of train batches %d" % len(train))
print("Building model...")
model = SequenceVAE(args.emb_dim,
args.hid_dim,
args.z_dim,
parser.add_argument('--word_dropout', default=0.0, type=float)
parser.add_argument('--learning_rate_decay', default=0.5, type=float)
parser.add_argument('--start_decay_at', default=8, type=int)
parser.add_argument('--max_grad_norm', default=5., type=float)
parser.add_argument('--gpu', action='store_true')
parser.add_argument('--beam', action='store_true')
args = parser.parse_args()
vocab = args.vocab
size = args.train_len
batch_size = args.batch_size
sample_fn = getattr(d, args.sample_fn)
if args.path != '':
with open(args.path, 'rb+') as f:
dataset = PairedDataset.from_disk(f)
dataset.set_batch_size(args.batch_size)
dataset.set_gpu(args.gpu)
train, valid = dataset.splits(sort_by='src', dev=args.dev, test=None)
src_dict = dataset.dicts['src']
else:
src, trg = zip(*d.generate_set(
size, vocab, args.min_len, args.max_len, sample_fn))
src, trg = list(map(list, src)), list(map(list, trg))
src_dict = Dict(pad_token=u.PAD, eos_token=u.EOS, bos_token=u.BOS)
src_dict.fit(src, trg)
train, valid = PairedDataset(
src, trg, {'src': src_dict, 'trg': src_dict},
batch_size=args.batch_size, gpu=args.gpu
).splits(dev=args.dev, test=None, sort_by='src')
parser.add_argument('--word_dropout', default=0.0, type=float)
parser.add_argument('--learning_rate_decay', default=0.5, type=float)
parser.add_argument('--start_decay_at', default=8, type=int)
parser.add_argument('--max_grad_norm', default=5., type=float)
parser.add_argument('--gpu', action='store_true')
parser.add_argument('--beam', action='store_true')
args = parser.parse_args()
vocab = args.vocab
size = args.train_len
batch_size = args.batch_size
sample_fn = getattr(d, args.sample_fn)
if args.path != '':
with open(args.path, 'rb+') as f:
dataset = PairedDataset.from_disk(f)
dataset.set_batch_size(args.batch_size)
dataset.set_gpu(args.gpu)
train, valid = dataset.splits(sort_by='src', dev=args.dev, test=None)
src_dict = dataset.dicts['src']
else:
src, trg = zip(*d.generate_set(size, vocab, args.min_len, args.max_len,
sample_fn))
src, trg = list(map(list, src)), list(map(list, trg))
src_dict = Dict(pad_token=u.PAD, eos_token=u.EOS, bos_token=u.BOS)
src_dict.fit(src, trg)
train, valid = PairedDataset(src,
trg, {
'src': src_dict,
'trg': src_dict
},
gpu=args.gpu, dev=args.dev, test=args.test)
elif args.source == 'penn':
train, test, valid = load_penn(
"~/corpora/penn", args.batch_size,
min_freq=args.min_freq, max_size=args.max_size, gpu=args.gpu)
else:
train, test, valid = load_from_lines(
args.source_path, args.batch_size,
min_freq=args.min_freq, max_size=args.max_size,
gpu=args.gpu, dev=args.dev, test=args.text)
if args.cache_data:
train.to_disk('data/%s_train.pt' % prefix)
test.to_disk('data/%s_test.pt' % prefix)
valid.to_disk('data/%s_valid.pt' % prefix)
else:
train = PairedDataset.from_disk('data/%s_train.pt' % prefix)
test = PairedDataset.from_disk('data/%s_test.pt' % prefix)
valid = PairedDataset.from_disk('data/%s_valid.pt' % prefix)
train.set_gpu(args.gpu), test.set_gpu(args.gpu), valid.set_gpu(args.gpu)
train.set_batch_size(args.batch_size), test.set_batch_size(args.batch_size)
valid.set_batch_size(args.batch_size)
datasets = {'train': train, 'valid': valid, 'test': test}
vocab = len(train.d['src'].vocab)
print("* Number of train batches %d" % len(train))
print("Building model...")
hid_dim = args.hid_dim if args.dec_hid_dim == 0 else (args.hid_dim, args.dec_hid_dim)
layers = args.layers if args.dec_layers == 0 else (args.layers, args.dec_layers)
model = SequenceVAE(
args.emb_dim, hid_dim, args.z_dim, train.d['src'],
num_layers=layers, cell=args.cell, bidi=not args.non_bidi,