def test_rnn(): np.random.seed(0) num_layers = 50 seq_length = num_layers * 2 batchsize = 2 vocab_size = 4 data = np.random.randint(0, vocab_size, size=(batchsize, seq_length), dtype=np.int32) source, target = make_source_target_pair(data) model = RNNModel(vocab_size, ndim_embedding=100, num_layers=num_layers, ndim_h=3, kernel_size=3, pooling="fo", zoneout=False, wgain=1) np.random.seed(0) model.reset_state() Y = model(source, test=True).data model.reset_state() np.random.seed(0) for t in xrange(source.shape[1]): y = model.forward_one_step(source[:, :t + 1], test=True).data target = np.swapaxes(np.reshape(Y, (batchsize, -1, vocab_size)), 1, 2) target = np.reshape(np.swapaxes(target[:, :, t, None], 1, 2), (batchsize, -1)) assert np.sum((y - target)**2) == 0 print("t = {} OK".format(t))
def main(args): # load textfile train_dataset, dev_dataset, test_dataset, vocab, vocab_inv = read_data( args.text_filename, train_split_ratio=args.train_split, dev_split_ratio=args.dev_split, seed=args.seed) save_vocab(args.model_dir, vocab, vocab_inv) vocab_size = len(vocab) print_bold("data # hash") print("train {} {}".format(len(train_dataset), hash(str(train_dataset)))) print("dev {} {}".format(len(dev_dataset), hash(str(dev_dataset)))) print("test {} {}".format(len(test_dataset), hash(str(test_dataset)))) print("vocab {}".format(vocab_size)) # split into buckets train_buckets = make_buckets(train_dataset) print_bold("buckets #data (train)") if args.buckets_limit is not None: train_buckets = train_buckets[:args.buckets_limit + 1] for size, data in zip(bucket_sizes, train_buckets): print("{} {}".format(size, len(data))) print_bold("buckets #data (dev)") dev_buckets = make_buckets(dev_dataset) if args.buckets_limit is not None: dev_buckets = dev_buckets[:args.buckets_limit + 1] for size, data in zip(bucket_sizes, dev_buckets): print("{} {}".format(size, len(data))) print_bold("buckets #data (test)") test_buckets = make_buckets(test_dataset) for size, data in zip(bucket_sizes, test_buckets): print("{} {}".format(size, len(data))) # to maintain equilibrium min_num_data = 0 for data in train_buckets: if min_num_data == 0 or len(data) < min_num_data: min_num_data = len(data) repeats = [] for data in train_buckets: repeat = len(data) // min_num_data repeat = repeat + 1 if repeat == 0 else repeat repeats.append(repeat) num_updates_per_iteration = 0 for repeat, data in zip(repeats, train_buckets): num_updates_per_iteration += repeat * args.batchsize num_iteration = len(train_dataset) // num_updates_per_iteration + 1 # init model = load_model(args.model_dir) if model is None: model = RNNModel(vocab_size, args.ndim_embedding, args.num_layers, ndim_h=args.ndim_h, kernel_size=args.kernel_size, pooling=args.pooling, zoneout=args.zoneout, dropout=args.dropout, wgain=args.wgain, densely_connected=args.densely_connected, ignore_label=ID_PAD) if args.gpu_device >= 0: chainer.cuda.get_device(args.gpu_device).use() model.to_gpu() # setup an optimizer if args.eve: optimizer = Eve(alpha=args.learning_rate, beta1=0.9) else: optimizer = optimizers.Adam(alpha=args.learning_rate, beta1=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip)) optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay)) min_learning_rate = 1e-7 prev_ppl = None total_time = 0 def mean(l): return sum(l) / len(l) # training for epoch in xrange(1, args.epoch + 1): print("Epoch", epoch) start_time = time.time() for itr in xrange(1, num_iteration + 1): sys.stdout.write("\r{} / {}".format(itr, num_iteration)) sys.stdout.flush() for repeat, dataset in zip(repeats, train_buckets): for r in xrange(repeat): batch = sample_batch_from_bucket(dataset, args.batchsize) source, target = make_source_target_pair(batch) if model.xp is cuda.cupy: source = cuda.to_gpu(source) target = cuda.to_gpu(target) model.reset_state() Y = model(source) loss = softmax_cross_entropy(Y, target, ignore_label=ID_PAD) optimizer.update(lossfun=lambda: loss) if itr % args.interval == 0 or itr == num_iteration: save_model(args.model_dir, model) # show log sys.stdout.write("\r" + stdout.CLEAR) sys.stdout.flush() print_bold(" accuracy (sampled train)") acc_train = compute_random_accuracy(model, train_buckets, args.batchsize) print(" ", mean(acc_train), acc_train) print_bold(" accuracy (dev)") acc_dev = compute_accuracy(model, dev_buckets, args.batchsize) print(" ", mean(acc_dev), acc_dev) print_bold(" ppl (sampled train)") ppl_train = compute_random_perplexity(model, train_buckets, args.batchsize) print(" ", mean(ppl_train), ppl_train) print_bold(" ppl (dev)") ppl_dev = compute_perplexity(model, dev_buckets, args.batchsize) ppl_dev_mean = mean(ppl_dev) print(" ", ppl_dev_mean, ppl_dev) elapsed_time = (time.time() - start_time) / 60. total_time += elapsed_time print(" done in {} min, lr = {}, total {} min".format( int(elapsed_time), optimizer.alpha, int(total_time))) # decay learning rate if prev_ppl is not None and ppl_dev_mean >= prev_ppl and optimizer.alpha > min_learning_rate: optimizer.alpha *= 0.5 prev_ppl = ppl_dev_mean
def main(): # load textfile dataset_train, dataset_dev, _, vocab, vocab_inv = read_data(args.train_filename, args.dev_filename) vocab_size = len(vocab) save_vocab(args.model_dir, vocab, vocab_inv) # split into buckets train_buckets = make_buckets(dataset_train) if args.buckets_slice is not None: train_buckets = train_buckets[:args.buckets_slice + 1] dev_buckets = None if len(dataset_dev) > 0: dev_buckets = make_buckets(dataset_dev) if args.buckets_slice is not None: dev_buckets = dev_buckets[:args.buckets_slice + 1] # print dump_dataset(dataset_train, dataset_dev, train_buckets, dev_buckets, vocab_size) # to maintain equilibrium required_interations = [] for data in train_buckets: itr = math.ceil(len(data) / args.batchsize) required_interations.append(itr) total_iterations = sum(required_interations) buckets_distribution = np.asarray(required_interations, dtype=float) / total_iterations # init model = load_model(args.model_dir) if model is None: model = RNNModel(vocab_size, args.ndim_embedding, args.num_layers, ndim_h=args.ndim_h, kernel_size=args.kernel_size, pooling=args.pooling, zoneout=args.zoneout, dropout=args.dropout, weightnorm=args.weightnorm, wgain=args.wgain, densely_connected=args.densely_connected, ignore_label=ID_PAD) if args.gpu_device >= 0: chainer.cuda.get_device(args.gpu_device).use() model.to_gpu() # setup an optimizer optimizer = get_optimizer(args.optimizer, args.learning_rate, args.momentum) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip)) optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay)) final_learning_rate = 1e-4 total_time = 0 def mean(l): return sum(l) / len(l) # training for epoch in range(1, args.epoch + 1): print("Epoch", epoch) start_time = time.time() with chainer.using_config("train", True): for itr in range(total_iterations): bucket_idx = int(np.random.choice(np.arange(len(train_buckets)), size=1, p=buckets_distribution)) dataset = train_buckets[bucket_idx] np.random.shuffle(dataset) data_batch = dataset[:args.batchsize] source_batch, target_batch = make_source_target_pair(data_batch) if args.gpu_device >= 0: source_batch = cuda.to_gpu(source_batch) target_batch = cuda.to_gpu(target_batch) # update params model.reset_state() y_batch = model(source_batch) loss = F.softmax_cross_entropy(y_batch, target_batch, ignore_label=ID_PAD) optimizer.update(lossfun=lambda: loss) # show log printr("iteration {}/{}".format(itr + 1, total_iterations)) save_model(args.model_dir, model) # clear console printr("") # compute perplexity with chainer.using_config("train", False): if dev_buckets is not None: printb(" ppl (dev)") ppl_dev = compute_perplexity(model, dev_buckets, args.batchsize) print(" ", mean(ppl_dev), ppl_dev) # show log elapsed_time = (time.time() - start_time) / 60. total_time += elapsed_time print(" done in {} min, lr = {}, total {} min".format(int(elapsed_time), get_current_learning_rate(optimizer), int(total_time))) # decay learning rate decay_learning_rate(optimizer, args.lr_decay_factor, final_learning_rate)