def get_model_scores(tasks_slice, payloads_slice, seed, eval_payload):
task_name = tasks_slice[0].name
slicing_tasks = convert_to_slicing_tasks(tasks_slice)
slice_model = SliceModel(slicing_tasks, seed=seed, verbose=False)
task_metrics = []
prefix = "{}/{}_".format(task_name, task_name)
for p_name, metric in [("train", "loss"), ("valid", "accuracy")]:
for label_name in eval_payload.labels_to_tasks.keys():
task_metrics += [
prefix + p_name + "/{}/{}".format(label_name, metric)
]
print(task_metrics)
trainer = MultitaskTrainer(seed=seed)
trainer.train_model(
slice_model,
payloads_slice,
task_metrics=task_metrics,
checkpoint_metric="{}/{}/{}_gold/accuracy".format(
task_name, eval_payload.name, task_name),
checkpoint_metric_mode="max",
checkoint_best=True,
writer="tensorboard",
optimizer="adamax",
lr=1e-5,
l2=1e-3,
log_every=0.1,
score_every=0.1,
n_epochs=20,
progress_bar=True,
checkpoint_best=True,
checkpoint_cleanup=False,
)
return slice_model.score(eval_payload)
def get_baseline_scores(tasks, seed, eval_payload):
# Baseline Model
task_name = tasks[0].name
model = MetalModel(tasks, seed=seed, verbose=False)
trainer = MultitaskTrainer(seed=seed)
trainer.train_model(
model,
payloads,
checkpoint_metric="{}/{}/{}_gold/accuracy".format(
task_name, eval_payload.name, task_name),
checkpoint_metric_mode="max",
checkoint_best=True,
writer="tensorboard",
optimizer="adamax",
lr=5e-5,
l2=1e-3,
log_every=0.1,
score_every=0.1,
n_epochs=10,
progress_bar=True,
checkpoint_best=True,
checkpoint_cleanup=False,
)
return model.score(eval_payload)
# "SST2",
# "MNLI",
# "RTE",
# "WNLI",
# "QQP",
# "MRPC",
# "STSB",
# "QNLI"
],
split_prop=0.99,
# max_datapoints = 100,
dl_kwargs={"batch_size": 2},
)
model = MetalModel(tasks)
trainer = MultitaskTrainer(**writer_dict)
trainer.train_model(
model,
payloads,
lr=0.00001,
l2=0,
log_every=1,
checkpoint_metric="model/train/loss",
checkpoint_metric_mode="min",
checkpoint_best=True,
n_epochs=3,
progress_bar=True,
)
print("WE WIN!")
payload_test = Payload.from_tensors(payload_test_name,
test_X,
test_Y,
task_name,
"test",
batch_size=batch_size)
payloads.append(payload_train)
payloads.append(payload_dev)
payloads.append(payload_test)
model = MetalModel(tasks, verbose=False)
os.environ['METALHOME'] = RESULTS_OUTPUT
trainer = MultitaskTrainer()
trainer.config["checkpoint_config"]["checkpoint_every"] = 1
trainer.config["checkpoint_config"]["checkpoint_best"] = True
for i in range(NUM_EPOCHS):
scores = trainer.train_model(
model,
payloads,
n_epochs=1,
log_every=1,
progress_bar=False,
)
print("Epoch number {0} scores:".format(i))
print(scores)
filename = OUTPUT_PATH + '/' + MODEL_NAME + '.out'
def setUpClass(cls):
np.random.seed(1)
cls.trainer = MultitaskTrainer(verbose=False, lr=0.005)
def test_mmtl_training(self):
model = MetalModel(self.tasks, verbose=False)
trainer = MultitaskTrainer(verbose=True)
trainer.train_model(
model, self.payloads, n_epochs=1, aggregate_metric_fns=[glue_score]
)
tasks, payloads = create_glue_tasks_payloads(task_names, **task_config)
model_config["verbose"] = False
model = MetalModel(tasks, **model_config)
# add metadata to trainer_config that will be logged to disk
trainer_config["n_paramaters"] = sum(p.numel() for p in model.parameters()
if p.requires_grad)
if trainer_config["verbose"]:
print(f"Task config:\n{task_config}")
print(f"Model config:\n{model_config}")
print(f"Trainer config:\n{trainer_config}")
# Overwrite run_dir to only use one checkpoint dir
# if args.run_dir is None:
# trainer_config["writer_config"]["run_dir"] = strftime("%Y_%m_%d")
if args.run_name is None:
trainer_config["writer_config"]["run_name"] = task_config[
"tasks"] # + strftime("_%H_%M_%S")
trainer = MultitaskTrainer(**trainer_config)
# Force early instantiation of writer to write all three configs to dict
trainer._set_writer()
# trainer_config will get written automatically right before training
trainer.writer.write_config(model_config, "model_config")
trainer.writer.write_config(task_config, "task_config")
trainer.train_model(model, payloads, aggregate_metric_fns=[glue_score])
"test",
batch_size=batch_size)
payloads.append(payload_train)
payloads.append(payload_dev)
payloads.append(payload_test)
return payloads
payloads = create_payloads(database_tasks)
model = MetalModel(tasks, verbose=False)
os.environ['METALHOME'] = RESULTS_OUTPUT
trainer = MultitaskTrainer()
trainer.config["checkpoint_config"]["checkpoint_every"] = 1
trainer.config["checkpoint_config"]["checkpoint_best"] = True
def accuracy(metrics_hist):
metrics_agg = {}
dev_accuracies = []
for key in sorted(metrics_hist.keys()):
if "accuracy" in key and "dev" in key:
dev_accuracies.append(metrics_hist[key])
average_accuracy = sum(dev_accuracies) / len(dev_accuracies)
metrics_agg["model/dev/all/accuracy"] = average_accuracy
bar_input = foo_input # nn.Linear(100, 7)
foo_head = nn.Linear(10, 2)
bar_head = nn.Linear(10, 2)
foo_data = make_dataloader(5000)
bar_data = make_dataloader(2000)
foo = Task("foo_task", {"train": foo_data, "valid": foo_data}, foo_input, foo_head)
bar = Task("bar_task", {"train": bar_data, "valid": bar_data}, bar_input, bar_head)
# baz = Task("baz_task", "baz_head", [make_dataloader(100), None, None])
tasks = [foo, bar]
model = MetalModel(tasks, device=-1, verbose=False)
trainer = MultitaskTrainer()
trainer.train_model(
model,
tasks,
n_epochs=3,
lr=0.1,
progress_bar=True,
log_every=1,
score_every=1,
checkpoint_best=True,
checkpoint_metric="foo_task/valid/accuracy",
checkpoint_metric_mode="max",
)
for batch in foo.data_loaders["train"]:
X, Y = batch