def build_batchers(word2id, cuda, debug): prepro = prepro_fn(args.max_art, args.max_abs) def sort_key(sample): src, target = sample return (len(target), len(src)) batchify = compose(batchify_fn_copy(PAD, START, END, cuda=cuda), convert_batch_copy(UNK, word2id)) train_loader = DataLoader(MatchDataset('train'), batch_size=BUCKET_SIZE, shuffle=not debug, num_workers=4 if cuda and not debug else 0, collate_fn=coll_fn) train_batcher = BucketedGenerater(train_loader, prepro, sort_key, batchify, single_run=False, fork=not debug) val_loader = DataLoader(MatchDataset('val'), batch_size=BUCKET_SIZE, shuffle=False, num_workers=4 if cuda and not debug else 0, collate_fn=coll_fn) val_batcher = BucketedGenerater(val_loader, prepro, sort_key, batchify, single_run=True, fork=not debug) return train_batcher, val_batcher
def build_batchers(word2id, cuda, debug): prepro = prepro_fn(args.max_art, args.max_abs) def sort_key(sample): src, target = sample return (len(target), len(src)) batchify = compose( batchify_fn_copy(PAD, START, END, cuda=cuda), convert_batch_copy(UNK, word2id) ) train_loader = DataLoader( MatchDataset('train'), batch_size=BUCKET_SIZE, shuffle=not debug, num_workers=4 if cuda and not debug else 0, collate_fn=coll_fn ) train_batcher = BucketedGenerater(train_loader, prepro, sort_key, batchify, single_run=False, fork=not debug) val_loader = DataLoader( MatchDataset('val'), batch_size=BUCKET_SIZE, shuffle=False, num_workers=4 if cuda and not debug else 0, collate_fn=coll_fn ) val_batcher = BucketedGenerater(val_loader, prepro, sort_key, batchify, single_run=True, fork=not debug) return train_batcher, val_batcher