def evaluate(model, data, params, last_save_file, split):
"""Evaluates a pretrained model on a dataset.
Inputs:
model (ATISModel): Model class.
data (ATISData): All of the data.
params (namespace): Parameters for the model.
last_save_file (str): Location where the model save file is.
"""
if last_save_file:
model.load(last_save_file)
else:
if not params.save_file:
raise ValueError(
"Must provide a save file name if not training first.")
model.load(params.save_file)
filename = split
if filename == 'dev':
split = data.dev_data
elif filename == 'train':
split = data.train_data
elif filename == 'test':
split = data.test_data
elif filename == 'valid':
split = data.valid_data
else:
raise ValueError("Split not recognized: " + str(params.evaluate_split))
if params.use_predicted_queries:
filename += "_use_predicted_queries"
else:
filename += "_use_gold_queries"
full_name = os.path.join(params.logdir, filename) + params.results_note
if params.interaction_level or params.use_predicted_queries:
examples = data.get_all_interactions(split)
if params.interaction_level:
evaluate_interaction_sample(
examples,
model,
name=full_name,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout,
use_predicted_queries=params.use_predicted_queries,
max_generation_length=params.eval_maximum_sql_length,
write_results=True,
use_gpu=True,
compute_metrics=params.compute_metrics)
else:
evaluate_using_predicted_queries(
examples,
model,
name=full_name,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout)
else:
examples = data.get_all_utterances(split)
evaluate_utterance_sample(
examples,
model,
name=full_name,
gold_forcing=False,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
max_generation_length=params.eval_maximum_sql_length,
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout,
write_results=True)
def train(model, data, params):
""" Trains a model.
Inputs:
model (ATISModel): The model to train.
data (ATISData): The data that is used to train.
params (namespace): Training parameters.
"""
# Get the training batches.
log = Logger(os.path.join(params.logdir, params.logfile), "w")
num_train_original = atis_data.num_utterances(data.train_data)
log.put("Original number of training utterances:\t"
+ str(num_train_original))
eval_fn = evaluate_utterance_sample
trainbatch_fn = data.get_utterance_batches
trainsample_fn = data.get_random_utterances
validsample_fn = data.get_all_utterances
batch_size = params.batch_size
if params.interaction_level:
batch_size = 1
eval_fn = evaluate_interaction_sample
trainbatch_fn = data.get_interaction_batches
trainsample_fn = data.get_random_interactions
validsample_fn = data.get_all_interactions
maximum_output_length = params.train_maximum_sql_length
train_batches = trainbatch_fn(batch_size,
max_output_length=maximum_output_length,
randomize=not params.deterministic)
if params.num_train >= 0:
train_batches = train_batches[:params.num_train]
training_sample = trainsample_fn(params.train_evaluation_size,
max_output_length=maximum_output_length)
valid_examples = validsample_fn(data.valid_data,
max_output_length=maximum_output_length)
num_train_examples = sum([len(batch) for batch in train_batches])
num_steps_per_epoch = len(train_batches)
log.put(
"Actual number of used training examples:\t" +
str(num_train_examples))
log.put("(Shortened by output limit of " +
str(maximum_output_length) +
")")
log.put("Number of steps per epoch:\t" + str(num_steps_per_epoch))
log.put("Batch size:\t" + str(batch_size))
print(
"Kept " +
str(num_train_examples) +
"/" +
str(num_train_original) +
" examples")
print(
"Batch size of " +
str(batch_size) +
" gives " +
str(num_steps_per_epoch) +
" steps per epoch")
# Keeping track of things during training.
epochs = 0
patience = params.initial_patience
learning_rate_coefficient = 1.
previous_epoch_loss = float('inf')
maximum_validation_accuracy = 0.
maximum_string_accuracy = 0.
countdown = int(patience)
if params.scheduler:
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model.trainer, mode='min', )
keep_training = True
while keep_training:
log.put("Epoch:\t" + str(epochs))
model.set_dropout(params.dropout_amount)
if not params.scheduler:
model.set_learning_rate(learning_rate_coefficient * params.initial_learning_rate)
# Run a training step.
if params.interaction_level:
epoch_loss = train_epoch_with_interactions(
train_batches,
params,
model,
randomize=not params.deterministic)
else:
epoch_loss = train_epoch_with_utterances(
train_batches,
model,
randomize=not params.deterministic)
log.put("train epoch loss:\t" + str(epoch_loss))
model.set_dropout(0.)
# Run an evaluation step on a sample of the training data.
train_eval_results = eval_fn(training_sample,
model,
params.train_maximum_sql_length,
name=os.path.join(params.logdir, "train-eval"),
write_results=True,
gold_forcing=True,
metrics=TRAIN_EVAL_METRICS)[0]
for name, value in train_eval_results.items():
log.put(
"train final gold-passing " +
name.name +
":\t" +
"%.2f" %
value)
# Run an evaluation step on the validation set.
valid_eval_results = eval_fn(valid_examples,
model,
params.eval_maximum_sql_length,
name=os.path.join(params.logdir, "valid-eval"),
write_results=True,
gold_forcing=True,
metrics=VALID_EVAL_METRICS)[0]
for name, value in valid_eval_results.items():
log.put("valid gold-passing " + name.name + ":\t" + "%.2f" % value)
valid_loss = valid_eval_results[Metrics.LOSS]
valid_token_accuracy = valid_eval_results[Metrics.TOKEN_ACCURACY]
string_accuracy = valid_eval_results[Metrics.STRING_ACCURACY]
if params.scheduler:
scheduler.step(valid_loss)
if valid_loss > previous_epoch_loss:
learning_rate_coefficient *= params.learning_rate_ratio
log.put(
"learning rate coefficient:\t" +
str(learning_rate_coefficient))
previous_epoch_loss = valid_loss
saved = False
if not saved and string_accuracy > maximum_string_accuracy:
maximum_string_accuracy = string_accuracy
patience = patience * params.patience_ratio
countdown = int(patience)
last_save_file = os.path.join(params.logdir, "save_" + str(epochs))
model.save(last_save_file)
log.put(
"maximum string accuracy:\t" +
str(maximum_string_accuracy))
log.put("patience:\t" + str(patience))
log.put("save file:\t" + str(last_save_file))
if countdown <= 0:
keep_training = False
countdown -= 1
log.put("countdown:\t" + str(countdown))
log.put("")
epochs += 1
log.put("Finished training!")
log.close()
return last_save_file
def evaluate(model, data, params, split):
"""Evaluates a pretrained model on a dataset.
Inputs:
model (ATISModel): Model class.
data (ATISData): All of the data.
params (namespace): Parameters for the model.
"""
filename = split
if filename == 'dev':
split = data.dev_data
elif filename == 'train':
split = data.train_data
elif filename == 'test':
split = data.test_data
elif filename == 'valid':
split = data.valid_data
else:
raise ValueError("Split not recognized: " + str(params.evaluate_split))
if params.use_predicted_queries:
filename += "_use_predicted_queries"
else:
filename += "_use_gold_queries"
full_name = os.path.join(params.logdir, filename) + params.results_note
if params.interaction_level or params.use_predicted_queries:
examples = data.get_all_interactions(split)
if params.interaction_level:
valid_eval_results = evaluate_interaction_sample(
examples,
model,
name=full_name,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout,
use_predicted_queries=params.use_predicted_queries,
max_generation_length=params.eval_maximum_sql_length,
write_results=True,
use_gpu=True,
compute_metrics=params.compute_metrics)[0]
else:
valid_eval_results = evaluate_using_predicted_queries(
examples,
model,
name=full_name,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout)[0]
else:
examples = data.get_all_utterances(split)
valid_eval_results = evaluate_utterance_sample(
examples,
model,
name=full_name,
gold_forcing=False,
metrics=FINAL_EVAL_METRICS,
total_num=atis_data.num_utterances(split),
max_generation_length=params.eval_maximum_sql_length,
database_username=params.database_username,
database_password=params.database_password,
database_timeout=params.database_timeout,
write_results=True)[0]
for name, value in valid_eval_results.items():
print("valid gold-passing " + name.name + ":\t" + "%.2f" % value)
valid_token_accuracy = valid_eval_results[Metrics.TOKEN_ACCURACY]
string_accuracy = valid_eval_results[Metrics.STRING_ACCURACY]
print("token accuracy:\t" + str(valid_token_accuracy))
print("maximum string accuracy:\t" + str(string_accuracy))
