def prepare_data(model, train_config, dataset_oversampling, n_processes):
extract = ExtractMultiParagraphsPerQuestion(
MergeParagraphs(train_config.n_tokens),
ShallowOpenWebRanker(train_config.num_paragraphs),
model.preprocessor, intern=True
)
trivia_qa_test = RandomParagraphSetDatasetBuilder(
train_config.test_batch_size,
"merge" if train_config.trivia_qa_mode == "merge" else "group", True,
train_config.oversample
)
trivia_qa_train = StratifyParagraphSetsBuilder(
train_config.train_batch_size,
train_config.trivia_qa_mode == "merge",
True, train_config.oversample
)
datas = []
for name, sampling in dataset_oversampling.items():
for s in range(sampling):
ds = TriviaQaSpanCorpus(name)
ds.corpus_name = ds.corpus_name + '_{}'.format(s)
datas.append(ds)
data = MultiDataset(datas)
data = PreprocessedData(data, extract, trivia_qa_train, trivia_qa_test, eval_on_verified=False)
data.preprocess(n_processes, 1000)
return data
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"corpus",
choices=["en", "fr", "de", "ru", "pt", "zh", "pl", "uk", "ta"])
parser.add_argument(
'mode',
choices=["confidence", "merge", "shared-norm", "sigmoid", "paragraph"])
# Note I haven't tested modes other than `shared-norm` on this corpus, so
# some things might need adjusting
parser.add_argument("-t",
"--n_tokens",
default=400,
type=int,
help="Paragraph size")
args = parser.parse_args()
mode = args.mode
corpus = args.corpus
model = get_model(100, 140, mode, WithIndicators())
extract = ExtractMultiParagraphsPerQuestion(MergeParagraphs(args.n_tokens),
ShallowOpenWebRanker(16),
model.preprocessor,
intern=True)
oversample = [
1
] * 2 # Sample the top two answer-containing paragraphs twice
if mode == "paragraph":
n_epochs = 120
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(ClusteredBatcher(
60, ContextLenBucketedKey(3), True),
oversample,
only_answers=True)
elif mode == "confidence" or mode == "sigmoid":
if mode == "sigmoid":
n_epochs = 640
else:
n_epochs = 160
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(
ClusteredBatcher(60, ContextLenBucketedKey(3), True), oversample)
else:
n_epochs = 80
test = RandomParagraphSetDatasetBuilder(
120, "merge" if mode == "merge" else "group", True, oversample)
train = StratifyParagraphSetsBuilder(30, mode == "merge", True,
oversample)
data = XQADataset(corpus)
data = PreprocessedData(data, extract, train, test, eval_on_verified=False)
data.preprocess(1, 1000)
# dump preprocessed dev data for bert
data.cache_preprocess("dev_data_%s.pkl" % args.corpus)
def main():
parser = argparse.ArgumentParser("Train our ELMo model on SQuAD")
parser.add_argument("loss_mode", choices=['default', 'confidence'])
parser.add_argument("output_dir")
parser.add_argument("--dim", type=int, default=90)
parser.add_argument("--l2", type=float, default=0)
parser.add_argument("--mode",
choices=["input", "output", "both", "none"],
default="both")
parser.add_argument("--top_layer_only", action="store_true")
parser.add_argument("--no-tfidf",
action='store_true',
help="Don't add TF-IDF negative examples")
args = parser.parse_args()
out = args.output_dir + "-" + datetime.now().strftime("%m%d-%H%M%S")
dim = args.dim
recurrent_layer = CudnnGru(dim, w_init=TruncatedNormal(stddev=0.05))
if args.loss_mode == 'default':
n_epochs = 24
answer_encoder = SingleSpanAnswerEncoder()
predictor = BoundsPredictor(
ChainBiMapper(first_layer=recurrent_layer,
second_layer=recurrent_layer))
batcher = ClusteredBatcher(45, ContextLenKey(), False, False)
data = DocumentQaTrainingData(SquadCorpus(), None, batcher, batcher)
elif args.loss_mode == 'confidence':
if args.no_tfidf:
prepro = SquadDefault()
n_epochs = 15
else:
prepro = SquadTfIdfRanker(NltkPlusStopWords(True), 4, True)
n_epochs = 50
answer_encoder = DenseMultiSpanAnswerEncoder()
predictor = ConfidencePredictor(ChainBiMapper(
first_layer=recurrent_layer,
second_layer=recurrent_layer,
),
AttentionEncoder(),
FullyConnected(80, activation="tanh"),
aggregate="sum")
eval_dataset = RandomParagraphSetDatasetBuilder(
100, 'flatten', True, 0)
train_batching = ClusteredBatcher(45, ContextLenBucketedKey(3), True,
False)
data = PreprocessedData(SquadCorpus(),
prepro,
StratifyParagraphsBuilder(train_batching, 1),
eval_dataset,
eval_on_verified=False)
data.preprocess(1)
params = trainer.TrainParams(trainer.SerializableOptimizer(
"Adadelta", dict(learning_rate=1.0)),
ema=0.999,
max_checkpoints_to_keep=2,
async_encoding=10,
num_epochs=n_epochs,
log_period=30,
eval_period=1200,
save_period=1200,
best_weights=("dev", "b17/text-f1"),
eval_samples=dict(dev=None, train=8000))
lm_reduce = MapperSeq(
ElmoLayer(args.l2,
layer_norm=False,
top_layer_only=args.top_layer_only),
DropoutLayer(0.5),
)
model = AttentionWithElmo(
encoder=DocumentAndQuestionEncoder(answer_encoder),
lm_model=SquadContextConcatSkip(),
append_before_atten=(args.mode == "both" or args.mode == "output"),
append_embed=(args.mode == "both" or args.mode == "input"),
max_batch_size=128,
word_embed=FixedWordEmbedder(vec_name="glove.840B.300d",
word_vec_init_scale=0,
learn_unk=False,
cpu=True),
char_embed=CharWordEmbedder(LearnedCharEmbedder(word_size_th=14,
char_th=49,
char_dim=20,
init_scale=0.05,
force_cpu=True),
MaxPool(Conv1d(100, 5, 0.8)),
shared_parameters=True),
embed_mapper=SequenceMapperSeq(
VariationalDropoutLayer(0.8),
recurrent_layer,
VariationalDropoutLayer(0.8),
),
lm_reduce=None,
lm_reduce_shared=lm_reduce,
per_sentence=False,
memory_builder=NullBiMapper(),
attention=BiAttention(TriLinear(bias=True), True),
match_encoder=SequenceMapperSeq(
FullyConnected(dim * 2, activation="relu"),
ResidualLayer(
SequenceMapperSeq(
VariationalDropoutLayer(0.8),
recurrent_layer,
VariationalDropoutLayer(0.8),
StaticAttentionSelf(TriLinear(bias=True),
ConcatWithProduct()),
FullyConnected(dim * 2, activation="relu"),
)), VariationalDropoutLayer(0.8)),
predictor=predictor)
with open(__file__, "r") as f:
notes = f.read()
notes = str(sorted(args.__dict__.items(),
key=lambda x: x[0])) + "\n" + notes
trainer.start_training(
data, model, params,
[LossEvaluator(),
SpanEvaluator(bound=[17], text_eval="squad")], ModelDir(out), notes)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("corpus", choices=["en", "en_trans_de", "en_trans_zh"])
parser.add_argument(
'mode',
choices=["confidence", "merge", "shared-norm", "sigmoid", "paragraph"])
# Note I haven't tested modes other than `shared-norm` on this corpus, so
# some things might need adjusting
parser.add_argument("name", help="Where to store the model")
parser.add_argument("-t",
"--n_tokens",
default=400,
type=int,
help="Paragraph size")
parser.add_argument(
'-n',
'--n_processes',
type=int,
default=2,
help="Number of processes (i.e., select which paragraphs to train on) "
"the data with")
args = parser.parse_args()
mode = args.mode
corpus = args.corpus
out = args.name + "-" + datetime.now().strftime("%m%d-%H%M%S")
model = get_model(100, 140, mode, WithIndicators())
extract = ExtractMultiParagraphsPerQuestion(MergeParagraphs(args.n_tokens),
ShallowOpenWebRanker(16),
model.preprocessor,
intern=True)
eval = [
LossEvaluator(),
MultiParagraphSpanEvaluator(8,
"triviaqa",
mode != "merge",
per_doc=False)
]
oversample = [
1
] * 2 # Sample the top two answer-containing paragraphs twice
if mode == "paragraph":
n_epochs = 120
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(ClusteredBatcher(
60, ContextLenBucketedKey(3), True),
oversample,
only_answers=True)
elif mode == "confidence" or mode == "sigmoid":
if mode == "sigmoid":
n_epochs = 640
else:
n_epochs = 160
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(
ClusteredBatcher(60, ContextLenBucketedKey(3), True), oversample)
else:
n_epochs = 80
test = RandomParagraphSetDatasetBuilder(
120, "merge" if mode == "merge" else "group", True, oversample)
train = StratifyParagraphSetsBuilder(30, mode == "merge", True,
oversample)
data = XQADataset(corpus)
params = TrainParams(SerializableOptimizer("Adadelta",
dict(learning_rate=1)),
num_epochs=n_epochs,
ema=0.999,
max_checkpoints_to_keep=2,
async_encoding=10,
log_period=30,
eval_period=1800,
save_period=1800,
best_weights=("dev", "b8/question-text-f1"),
eval_samples=dict(dev=None, train=6000))
data = PreprocessedData(data, extract, train, test, eval_on_verified=False)
data.preprocess(args.n_processes, 1000)
with open(__file__, "r") as f:
notes = f.read()
notes = "Mode: " + args.mode + "\n" + notes
trainer.start_training(data, model, params, eval, model_dir.ModelDir(out),
notes)
def main():
parser = argparse.ArgumentParser(description='Train a model on TriviaQA web')
parser.add_argument('mode', choices=["paragraph-level", "confidence", "merge",
"shared-norm", "sigmoid", "shared-norm-600"])
parser.add_argument("name", help="Where to store the model")
parser.add_argument('-n', '--n_processes', type=int, default=2,
help="Number of processes (i.e., select which paragraphs to train on) "
"the data with")
args = parser.parse_args()
mode = args.mode
out = args.name + "-" + datetime.now().strftime("%m%d-%H%M%S")
model = get_model(100, 140, mode, WithIndicators())
stop = NltkPlusStopWords(True)
if mode == "paragraph-level":
extract = ExtractSingleParagraph(MergeParagraphs(400), TopTfIdf(stop, 1), model.preprocessor, intern=True)
elif mode == "shared-norm-600":
extract = ExtractMultiParagraphs(MergeParagraphs(600), TopTfIdf(stop, 4), model.preprocessor, intern=True)
else:
extract = ExtractMultiParagraphs(MergeParagraphs(400), TopTfIdf(stop, 4), model.preprocessor, intern=True)
if mode == "paragraph-level":
n_epochs = 16
train = ParagraphAndQuestionsBuilder(ClusteredBatcher(60, ContextLenBucketedKey(3), True))
test = ParagraphAndQuestionsBuilder(ClusteredBatcher(60, ContextLenKey(), False))
n_dev, n_train = 21000, 12000
eval = [LossEvaluator(), SpanEvaluator([4, 8], "triviaqa")]
else:
eval = [LossEvaluator(), MultiParagraphSpanEvaluator(8, "triviaqa", mode != "merge")]
# we sample two paragraphs per a (question, doc) pair, so evaluate on fewer questions
n_dev, n_train = 15000, 8000
if mode == "confidence" or mode == "sigmoid":
if mode == "sigmoid":
# Trains very slowly, do this at your own risk
n_epochs = 71
else:
n_epochs = 28
test = RandomParagraphSetDatasetBuilder(120, "flatten", True, 1)
train = StratifyParagraphsBuilder(ClusteredBatcher(60, ContextLenBucketedKey(3), True), 0, 1)
else:
n_epochs = 14
test = RandomParagraphSetDatasetBuilder(120, "merge" if mode == "merge" else "group", True, 1)
train = StratifyParagraphSetsBuilder(35, mode == "merge", True, 1)
data = TriviaQaWebDataset()
params = get_triviaqa_train_params(n_epochs, n_dev, n_train)
data = PreprocessedData(data, extract, train, test, eval_on_verified=False)
data.preprocess(args.n_processes, 1000)
with open(__file__, "r") as f:
notes = f.read()
notes = "*" * 10 + "\nMode: " + args.mode + "\n" + "*"*10 + "\n" + notes
trainer.start_training(data, model, params, eval, model_dir.ModelDir(out), notes)
def main():
parser = argparse.ArgumentParser(
description='Train a model on document-level SQuAD')
parser.add_argument(
'mode',
choices=["paragraph", "confidence", "shared-norm", "merge", "sigmoid"])
parser.add_argument("name", help="Output directory")
args = parser.parse_args()
mode = args.mode
out = args.name + "-" + datetime.now().strftime("%m%d-%H%M%S")
corpus = SquadCorpus()
if mode == "merge":
# Adds paragraph start tokens, since we will be concatenating paragraphs together
pre = WithIndicators(True, para_tokens=False, doc_start_token=False)
else:
pre = None
model = get_model(50, 100, args.mode, pre)
if mode == "paragraph":
# Run in the "standard" known-paragraph setting
if model.preprocessor is not None:
raise NotImplementedError()
n_epochs = 26
train_batching = ClusteredBatcher(45, ContextLenBucketedKey(3), True,
False)
eval_batching = ClusteredBatcher(45, ContextLenKey(), False, False)
data = DocumentQaTrainingData(corpus, None, train_batching,
eval_batching)
eval = [LossEvaluator(), SpanEvaluator(bound=[17], text_eval="squad")]
else:
eval_set_mode = {
"confidence": "flatten",
"sigmoid": "flatten",
"shared-norm": "group",
"merge": "merge"
}[mode]
eval_dataset = RandomParagraphSetDatasetBuilder(
100, eval_set_mode, True, 0)
if mode == "confidence" or mode == "sigmoid":
if mode == "sigmoid":
# needs to be trained for a really long time for reasons unknown, even this might be too small
n_epochs = 100
else:
n_epochs = 50 # more epochs since we only "see" the label very other epoch-osh
train_batching = ClusteredBatcher(45, ContextLenBucketedKey(3),
True, False)
data = PreprocessedData(
SquadCorpus(),
SquadTfIdfRanker(NltkPlusStopWords(True), 4, True,
model.preprocessor),
StratifyParagraphsBuilder(train_batching, 1),
eval_dataset,
eval_on_verified=False,
)
else:
n_epochs = 26
data = PreprocessedData(
SquadCorpus(),
SquadTfIdfRanker(NltkPlusStopWords(True), 4, True,
model.preprocessor),
StratifyParagraphSetsBuilder(25, args.mode == "merge", True,
1),
eval_dataset,
eval_on_verified=False,
)
eval = [LossEvaluator(), MultiParagraphSpanEvaluator(17, "squad")]
data.preprocess(1)
with open(__file__, "r") as f:
notes = f.read()
notes = args.mode + "\n" + notes
trainer.start_training(data, model, train_params(n_epochs), eval,
model_dir.ModelDir(out), notes)
def main():
parser = argparse.ArgumentParser(
description='Train a model on TriviaQA unfiltered')
parser.add_argument(
'mode',
choices=["confidence", "merge", "shared-norm", "sigmoid", "paragraph"])
parser.add_argument("name", help="Where to store the model")
parser.add_argument("-t",
"--n_tokens",
default=400,
type=int,
help="Paragraph size")
parser.add_argument(
'-n',
'--n_processes',
type=int,
default=2,
help="Number of processes (i.e., select which paragraphs to train on) "
"the data with")
parser.add_argument("-s",
"--source_dir",
type=str,
default=None,
help="where to take input files")
parser.add_argument("--n_epochs",
type=int,
default=None,
help="Max number of epoches to train on ")
parser.add_argument("--char_th",
type=int,
default=None,
help="char level embeddings")
parser.add_argument("--hl_dim",
type=int,
default=None,
help="hidden layer dim size")
parser.add_argument("--regularization",
type=int,
default=None,
help="hidden layer dim size")
parser.add_argument("--LR",
type=float,
default=1.0,
help="hidden layer dim size")
parser.add_argument("--save_every",
type=int,
default=1800,
help="save period")
parser.add_argument("--init_from",
type=str,
default=None,
help="model to init from")
args = parser.parse_args()
mode = args.mode
#out = args.name + "-" + datetime.now().strftime("%m%d-%H%M%S")
out = join('models', args.name)
char_th = 100
hl_dim = 140
if args.char_th is not None:
print(args.char_th)
char_th = int(args.char_th)
out += '--th' + str(char_th)
if args.hl_dim is not None:
print(args.hl_dim)
hl_dim = int(args.hl_dim)
out += '--hl' + str(hl_dim)
if args.init_from is None:
model = get_model(char_th, hl_dim, mode, WithIndicators())
else:
md = model_dir.ModelDir(args.init_from)
model = md.get_model()
extract = ExtractMultiParagraphsPerQuestion(MergeParagraphs(args.n_tokens),
ShallowOpenWebRanker(16),
model.preprocessor,
intern=True)
eval = [
LossEvaluator(),
MultiParagraphSpanEvaluator(8,
"triviaqa",
mode != "merge",
per_doc=False)
]
oversample = [1] * 4
if mode == "paragraph":
n_epochs = 120
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(ClusteredBatcher(
60, ContextLenBucketedKey(3), True),
oversample,
only_answers=True)
elif mode == "confidence" or mode == "sigmoid":
if mode == "sigmoid":
n_epochs = 640
else:
n_epochs = 160
test = RandomParagraphSetDatasetBuilder(120, "flatten", True,
oversample)
train = StratifyParagraphsBuilder(
ClusteredBatcher(60, ContextLenBucketedKey(3), True), oversample)
else:
n_epochs = 80
test = RandomParagraphSetDatasetBuilder(
120, "merge" if mode == "merge" else "group", True, oversample)
train = StratifyParagraphSetsBuilder(30, mode == "merge", True,
oversample)
if args.n_epochs is not None:
n_epochs = args.n_epochs
out += '--' + str(n_epochs)
if args.LR != 1.0:
out += '--' + str(args.LR)
data = TriviaQaOpenDataset(args.source_dir)
async_encoding = 10
#async_encoding = 0
params = TrainParams(SerializableOptimizer("Adadelta",
dict(learning_rate=args.LR)),
num_epochs=n_epochs,
num_of_steps=250000,
ema=0.999,
max_checkpoints_to_keep=2,
async_encoding=async_encoding,
log_period=30,
eval_period=1800,
save_period=args.save_every,
eval_samples=dict(dev=None, train=6000),
regularization_weight=None)
data = PreprocessedData(data, extract, train, test, eval_on_verified=False)
data.preprocess(args.n_processes, 1000)
with open(__file__, "r") as f:
notes = f.read()
notes = "Mode: " + args.mode + "\n" + notes
if args.init_from is not None:
init_from = model_dir.ModelDir(args.init_from).get_best_weights()
if init_from is None:
init_from = model_dir.ModelDir(
args.init_from).get_latest_checkpoint()
else:
init_from = None
trainer.start_training(data,
model,
params,
eval,
model_dir.ModelDir(out),
notes,
initialize_from=init_from)
parser.add_argument('--num_epoch',
required=True,
type=int,
help="num_epoch, e.g. 10")
args = parser.parse_args()
mode = "shared-norm"
model = get_model(100, 140, mode, WithIndicators())
extract = ExtractMultiParagraphsPerQuestion(MergeParagraphs(400),
ShallowOpenWebRanker(16),
model.preprocessor,
intern=True)
oversample = [1] * 2
train = StratifyParagraphSetsBuilder(30, mode == "merge", True, oversample)
test = RandomParagraphSetDatasetBuilder(
120, "merge" if mode == "merge" else "group", True, oversample)
data = PreprocessedData(None, extract, train, test, eval_on_verified=False)
data.load_preprocess(args.input_file)
outputs = []
training_data = data.get_train()
for i in range(args.num_epoch):
for batch in training_data.get_epoch():
last_qid = None
current = {
"question_id": "",
"question": [],
"context": [],
"answer_spans": [],
"answer_text": []