def main(args):
# load textfile
train_dataset, dev_dataset, test_dataset, vocab, vocab_inv = read_data(
args.text_filename,
train_split_ratio=args.train_split,
dev_split_ratio=args.dev_split,
seed=args.seed)
save_vocab(args.model_dir, vocab, vocab_inv)
vocab_size = len(vocab)
print_bold("data # hash")
print("train {} {}".format(len(train_dataset), hash(str(train_dataset))))
print("dev {} {}".format(len(dev_dataset), hash(str(dev_dataset))))
print("test {} {}".format(len(test_dataset), hash(str(test_dataset))))
print("vocab {}".format(vocab_size))
# split into buckets
train_buckets = make_buckets(train_dataset)
print_bold("buckets #data (train)")
if args.buckets_limit is not None:
train_buckets = train_buckets[:args.buckets_limit + 1]
for size, data in zip(bucket_sizes, train_buckets):
print("{} {}".format(size, len(data)))
print_bold("buckets #data (dev)")
dev_buckets = make_buckets(dev_dataset)
if args.buckets_limit is not None:
dev_buckets = dev_buckets[:args.buckets_limit + 1]
for size, data in zip(bucket_sizes, dev_buckets):
print("{} {}".format(size, len(data)))
print_bold("buckets #data (test)")
test_buckets = make_buckets(test_dataset)
for size, data in zip(bucket_sizes, test_buckets):
print("{} {}".format(size, len(data)))
# to maintain equilibrium
min_num_data = 0
for data in train_buckets:
if min_num_data == 0 or len(data) < min_num_data:
min_num_data = len(data)
repeats = []
for data in train_buckets:
repeat = len(data) // min_num_data
repeat = repeat + 1 if repeat == 0 else repeat
repeats.append(repeat)
num_updates_per_iteration = 0
for repeat, data in zip(repeats, train_buckets):
num_updates_per_iteration += repeat * args.batchsize
num_iteration = len(train_dataset) // num_updates_per_iteration + 1
# init
model = load_model(args.model_dir)
if model is None:
model = RNNModel(vocab_size,
args.ndim_embedding,
args.num_layers,
ndim_h=args.ndim_h,
kernel_size=args.kernel_size,
pooling=args.pooling,
zoneout=args.zoneout,
dropout=args.dropout,
wgain=args.wgain,
densely_connected=args.densely_connected,
ignore_label=ID_PAD)
if args.gpu_device >= 0:
chainer.cuda.get_device(args.gpu_device).use()
model.to_gpu()
# setup an optimizer
if args.eve:
optimizer = Eve(alpha=args.learning_rate, beta1=0.9)
else:
optimizer = optimizers.Adam(alpha=args.learning_rate, beta1=0.9)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip))
optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay))
min_learning_rate = 1e-7
prev_ppl = None
total_time = 0
def mean(l):
return sum(l) / len(l)
# training
for epoch in xrange(1, args.epoch + 1):
print("Epoch", epoch)
start_time = time.time()
for itr in xrange(1, num_iteration + 1):
sys.stdout.write("\r{} / {}".format(itr, num_iteration))
sys.stdout.flush()
for repeat, dataset in zip(repeats, train_buckets):
for r in xrange(repeat):
batch = sample_batch_from_bucket(dataset, args.batchsize)
source, target = make_source_target_pair(batch)
if model.xp is cuda.cupy:
source = cuda.to_gpu(source)
target = cuda.to_gpu(target)
model.reset_state()
Y = model(source)
loss = softmax_cross_entropy(Y,
target,
ignore_label=ID_PAD)
optimizer.update(lossfun=lambda: loss)
if itr % args.interval == 0 or itr == num_iteration:
save_model(args.model_dir, model)
# show log
sys.stdout.write("\r" + stdout.CLEAR)
sys.stdout.flush()
print_bold(" accuracy (sampled train)")
acc_train = compute_random_accuracy(model, train_buckets,
args.batchsize)
print(" ", mean(acc_train), acc_train)
print_bold(" accuracy (dev)")
acc_dev = compute_accuracy(model, dev_buckets, args.batchsize)
print(" ", mean(acc_dev), acc_dev)
print_bold(" ppl (sampled train)")
ppl_train = compute_random_perplexity(model, train_buckets,
args.batchsize)
print(" ", mean(ppl_train), ppl_train)
print_bold(" ppl (dev)")
ppl_dev = compute_perplexity(model, dev_buckets, args.batchsize)
ppl_dev_mean = mean(ppl_dev)
print(" ", ppl_dev_mean, ppl_dev)
elapsed_time = (time.time() - start_time) / 60.
total_time += elapsed_time
print(" done in {} min, lr = {}, total {} min".format(
int(elapsed_time), optimizer.alpha, int(total_time)))
# decay learning rate
if prev_ppl is not None and ppl_dev_mean >= prev_ppl and optimizer.alpha > min_learning_rate:
optimizer.alpha *= 0.5
prev_ppl = ppl_dev_mean
def main():
# load textfile
dataset_train, dataset_dev, _, vocab, vocab_inv = read_data(args.train_filename, args.dev_filename)
vocab_size = len(vocab)
save_vocab(args.model_dir, vocab, vocab_inv)
# split into buckets
train_buckets = make_buckets(dataset_train)
if args.buckets_slice is not None:
train_buckets = train_buckets[:args.buckets_slice + 1]
dev_buckets = None
if len(dataset_dev) > 0:
dev_buckets = make_buckets(dataset_dev)
if args.buckets_slice is not None:
dev_buckets = dev_buckets[:args.buckets_slice + 1]
# print
dump_dataset(dataset_train, dataset_dev, train_buckets, dev_buckets, vocab_size)
# to maintain equilibrium
required_interations = []
for data in train_buckets:
itr = math.ceil(len(data) / args.batchsize)
required_interations.append(itr)
total_iterations = sum(required_interations)
buckets_distribution = np.asarray(required_interations, dtype=float) / total_iterations
# init
model = load_model(args.model_dir)
if model is None:
model = RNNModel(vocab_size, args.ndim_embedding, args.num_layers, ndim_h=args.ndim_h, kernel_size=args.kernel_size, pooling=args.pooling, zoneout=args.zoneout, dropout=args.dropout, weightnorm=args.weightnorm, wgain=args.wgain, densely_connected=args.densely_connected, ignore_label=ID_PAD)
if args.gpu_device >= 0:
chainer.cuda.get_device(args.gpu_device).use()
model.to_gpu()
# setup an optimizer
optimizer = get_optimizer(args.optimizer, args.learning_rate, args.momentum)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip))
optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay))
final_learning_rate = 1e-4
total_time = 0
def mean(l):
return sum(l) / len(l)
# training
for epoch in range(1, args.epoch + 1):
print("Epoch", epoch)
start_time = time.time()
with chainer.using_config("train", True):
for itr in range(total_iterations):
bucket_idx = int(np.random.choice(np.arange(len(train_buckets)), size=1, p=buckets_distribution))
dataset = train_buckets[bucket_idx]
np.random.shuffle(dataset)
data_batch = dataset[:args.batchsize]
source_batch, target_batch = make_source_target_pair(data_batch)
if args.gpu_device >= 0:
source_batch = cuda.to_gpu(source_batch)
target_batch = cuda.to_gpu(target_batch)
# update params
model.reset_state()
y_batch = model(source_batch)
loss = F.softmax_cross_entropy(y_batch, target_batch, ignore_label=ID_PAD)
optimizer.update(lossfun=lambda: loss)
# show log
printr("iteration {}/{}".format(itr + 1, total_iterations))
save_model(args.model_dir, model)
# clear console
printr("")
# compute perplexity
with chainer.using_config("train", False):
if dev_buckets is not None:
printb(" ppl (dev)")
ppl_dev = compute_perplexity(model, dev_buckets, args.batchsize)
print(" ", mean(ppl_dev), ppl_dev)
# show log
elapsed_time = (time.time() - start_time) / 60.
total_time += elapsed_time
print(" done in {} min, lr = {}, total {} min".format(int(elapsed_time), get_current_learning_rate(optimizer), int(total_time)))
# decay learning rate
decay_learning_rate(optimizer, args.lr_decay_factor, final_learning_rate)
