def example_usage_full():
# 1. define model
model = torch.nn.Sequential(torch.nn.Linear(5, 5), torch.nn.Softmax(-1))
# 2. define data
x = torch.rand(100, 5)
y = torch.randint(0, 5, (100, ))
dataset = torch.utils.data.TensorDataset(x, y)
traindataset, validdataset, testdataset = torch.utils.data.random_split(
dataset, [70, 10, 20])
trainloader = torch.utils.data.DataLoader(traindataset,
batch_size=2,
shuffle=True)
validloader = torch.utils.data.DataLoader(validdataset,
batch_size=2,
shuffle=False)
testloader = torch.utils.data.DataLoader(testdataset,
batch_size=2,
shuffle=False)
# 3. define losses and wrap them
loss = torch.nn.CrossEntropyLoss(reduction="mean")
loss2 = torch.nn.CrossEntropyLoss(reduction="sum")
loss = q.LossWrapper(loss)
loss2 = q.LossWrapper(loss2)
# 4. define optim
optim = torch.optim.SGD(model.parameters(), lr=1.0)
# 5. other options (device, ...)
device = torch.device("cpu")
# 6. define training function (using partial)
trainepoch = partial(q.train_epoch,
model=model,
dataloader=trainloader,
optim=optim,
losses=[loss, loss2],
device=device)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=model,
dataloader=validloader,
losses=[loss, loss2],
device=device)
# 8. run training
run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=50)
# 9. run test function
testresults = q.test_epoch(model=model,
dataloader=testloader,
losses=[loss, loss2],
device=device)
print(testresults)
def run(
lr=0.001,
batsize=20,
epochs=70,
embdim=128,
encdim=400,
numlayers=1,
beamsize=5,
dropout=.5,
wreg=1e-10,
cuda=False,
gpu=0,
minfreq=2,
gradnorm=3.,
smoothing=0.1,
cosine_restarts=1.,
seed=123456,
):
localargs = locals().copy()
print(locals())
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if not cuda else torch.device("cuda", gpu)
tt.tick("loading data")
ds = GeoDatasetRank()
print(
f"max lens: {ds.maxlen_input} (input) and {ds.maxlen_output} (output)")
tt.tock("data loaded")
# do_rare_stats(ds)
# model = TreeRankModel(embdim=embdim, hdim=encdim, dropout=dropout, numlayers=numlayers,
# sentence_encoder=ds.sentence_encoder, query_encoder=ds.query_encoder)
#
model = ParikhRankModel(embdim=encdim,
dropout=dropout,
sentence_encoder=ds.sentence_encoder,
query_encoder=ds.query_encoder)
# sentence_rare_tokens = set([ds.sentence_encoder.vocab(i) for i in model.inp_emb.rare_token_ids])
# do_rare_stats(ds, sentence_rare_tokens=sentence_rare_tokens)
ranker = Ranker(model,
eval=[BCELoss(mode="logits", smoothing=smoothing)],
evalseq=[
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(
tensor2tree, D=ds.query_encoder.vocab),
orderless={"and", "or"})
])
losses = make_array_of_metrics("loss", "seq_acc", "tree_acc")
vlosses = make_array_of_metrics("seq_acc", "tree_acc")
# 4. define optim
# optim = torch.optim.Adam(trainable_params, lr=lr, weight_decay=wreg)
optim = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wreg)
# lr schedule
if cosine_restarts >= 0:
# t_max = epochs * len(train_dl)
t_max = epochs
print(f"Total number of updates: {t_max}")
lr_schedule = q.WarmupCosineWithHardRestartsSchedule(
optim, 0, t_max, cycles=cosine_restarts)
reduce_lr = [lambda: lr_schedule.step()]
else:
reduce_lr = []
# 6. define training function
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
model.parameters(), gradnorm)
# clipgradnorm = lambda: None
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=ranker,
dataloader=ds.dataloader("train", batsize),
optim=optim,
losses=losses,
_train_batch=trainbatch,
device=device,
on_end=reduce_lr)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=ranker,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
# 7. run training
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs)
tt.tock("done training")
# testing
tt.tick("testing")
testresults = q.test_epoch(model=ranker,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print("validation test results: ", testresults)
tt.tock("tested")
tt.tick("testing")
testresults = q.test_epoch(model=ranker,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print("test results: ", testresults)
tt.tock("tested")
# save model?
tosave = input(
"Save this model? 'y(es)'=Yes, <int>=overwrite previous, otherwise=No) \n>"
)
# if True:
# overwrite = None
if tosave.lower() == "y" or tosave.lower() == "yes" or re.match(
"\d+", tosave.lower()):
overwrite = int(tosave) if re.match("\d+", tosave) else None
p = q.save_run(model,
localargs,
filepath=__file__,
overwrite=overwrite)
q.save_dataset(ds, p)
_model, _localargs = q.load_run(p)
_ds = q.load_dataset(p)
_freedecoder = BeamDecoder(
_model,
maxtime=100,
beamsize=beamsize,
copy_deep=True,
eval=[SeqAccuracies()],
eval_beam=[
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})
])
# testing
tt.tick("testing reloaded")
_testresults = q.test_epoch(model=_freedecoder,
dataloader=_ds.dataloader("test", batsize),
losses=beamlosses,
device=device)
print(_testresults)
tt.tock("tested")
# save predictions
_, testpreds = q.eval_loop(_freedecoder,
ds.dataloader("test",
batsize=batsize,
shuffle=False),
device=device)
testout = get_outputs_for_save(testpreds)
_, trainpreds = q.eval_loop(_freedecoder,
ds.dataloader("train",
batsize=batsize,
shuffle=False),
device=device)
trainout = get_outputs_for_save(trainpreds)
with open(os.path.join(p, "trainpreds.json"), "w") as f:
ujson.dump(trainout, f)
with open(os.path.join(p, "testpreds.json"), "w") as f:
ujson.dump(testout, f)
def run(
lr=0.001,
batsize=50,
epochs=50,
embdim=100,
encdim=100,
numlayers=1,
beamsize=1,
dropout=.2,
wreg=1e-10,
cuda=False,
gpu=0,
minfreq=3,
gradnorm=3.,
cosine_restarts=1.,
beta=0.001,
vib_init=True,
vib_enc=True,
):
localargs = locals().copy()
print(locals())
tt = q.ticktock("script")
device = torch.device("cpu") if not cuda else torch.device("cuda", gpu)
tt.tick("loading data")
ds = LCQuaDnoENTDataset(
sentence_encoder=SequenceEncoder(tokenizer=split_tokenizer),
min_freq=minfreq)
print(
f"max lens: {ds.maxlen_input} (input) and {ds.maxlen_output} (output)")
tt.tock("data loaded")
do_rare_stats(ds)
# batch = next(iter(train_dl))
# print(batch)
# print("input graph")
# print(batch.batched_states)
model = BasicGenModel_VIB(embdim=embdim,
hdim=encdim,
dropout=dropout,
numlayers=numlayers,
sentence_encoder=ds.sentence_encoder,
query_encoder=ds.query_encoder,
feedatt=True,
vib_init=vib_init,
vib_enc=vib_enc)
# sentence_rare_tokens = set([ds.sentence_encoder.vocab(i) for i in model.inp_emb.rare_token_ids])
# do_rare_stats(ds, sentence_rare_tokens=sentence_rare_tokens)
losses = [CELoss(ignore_index=0, mode="logprobs")]
if vib_init:
losses.append(
StatePenalty(lambda state: sum(state.mstate.vib.init),
weight=beta))
if vib_enc:
losses.append(StatePenalty("mstate.vib.enc", weight=beta))
tfdecoder = SeqDecoder(
model,
tf_ratio=1.,
eval=losses + [
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"select", "count", "ask"})
])
# beamdecoder = BeamActionSeqDecoder(tfdecoder.model, beamsize=beamsize, maxsteps=50)
if beamsize == 1:
freedecoder = SeqDecoder(
model,
maxtime=40,
tf_ratio=0.,
eval=[
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"select", "count", "ask"})
])
else:
freedecoder = BeamDecoder(
model,
maxtime=30,
beamsize=beamsize,
eval=[
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"select", "count", "ask"})
])
# # test
# tt.tick("doing one epoch")
# for batch in iter(train_dl):
# batch = batch.to(device)
# ttt.tick("start batch")
# # with torch.no_grad():
# out = tfdecoder(batch)
# ttt.tock("end batch")
# tt.tock("done one epoch")
# print(out)
# sys.exit()
# beamdecoder(next(iter(train_dl)))
# print(dict(tfdecoder.named_parameters()).keys())
losses = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
vlosses = make_array_of_metrics("seq_acc", "tree_acc")
# if beamsize >= 3:
# vlosses = make_loss_array("seq_acc", "tree_acc", "tree_acc_at3", "tree_acc_at_last")
# else:
# vlosses = make_loss_array("seq_acc", "tree_acc", "tree_acc_at_last")
# trainable_params = tfdecoder.named_parameters()
# exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
# trainable_params = [v for k, v in trainable_params if k not in exclude_params]
# 4. define optim
# optim = torch.optim.Adam(trainable_params, lr=lr, weight_decay=wreg)
optim = torch.optim.Adam(tfdecoder.parameters(), lr=lr, weight_decay=wreg)
# lr schedule
if cosine_restarts >= 0:
# t_max = epochs * len(train_dl)
t_max = epochs
print(f"Total number of updates: {t_max}")
lr_schedule = q.WarmupCosineWithHardRestartsSchedule(
optim, 0, t_max, cycles=cosine_restarts)
reduce_lr = [lambda: lr_schedule.step()]
else:
reduce_lr = []
# 6. define training function
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
tfdecoder.parameters(), gradnorm)
# clipgradnorm = lambda: None
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=tfdecoder,
dataloader=ds.dataloader("train", batsize),
optim=optim,
losses=losses,
_train_batch=trainbatch,
device=device,
on_end=reduce_lr)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=freedecoder,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
# validepoch = partial(q.test_epoch, model=freedecoder, dataloader=valid_dl, losses=vlosses, device=device)
# p = q.save_run(freedecoder, localargs, filepath=__file__)
# q.save_dataset(ds, p)
# _freedecoder, _localargs = q.load_run(p)
# _ds = q.load_dataset(p)
# sys.exit()
# 7. run training
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs)
tt.tock("done training")
# testing
tt.tick("testing")
testresults = q.test_epoch(model=freedecoder,
dataloader=ds.dataloader("valid", batsize),
losses=vlosses,
device=device)
print("validation test results: ", testresults)
tt.tock("tested")
tt.tick("testing")
testresults = q.test_epoch(model=freedecoder,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print("test results: ", testresults)
tt.tock("tested")
# save model?
tosave = input(
"Save this model? 'y(es)'=Yes, <int>=overwrite previous, otherwise=No) \n>"
)
if tosave.lower() == "y" or tosave.lower() == "yes" or re.match(
"\d+", tosave.lower()):
overwrite = int(tosave) if re.match("\d+", tosave) else None
p = q.save_run(model,
localargs,
filepath=__file__,
overwrite=overwrite)
q.save_dataset(ds, p)
_model, _localargs = q.load_run(p)
_ds = q.load_dataset(p)
_freedecoder = BeamDecoder(
_model,
maxtime=50,
beamsize=beamsize,
eval_beam=[
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})
])
# testing
tt.tick("testing reloaded")
_testresults = q.test_epoch(model=_freedecoder,
dataloader=_ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print(_testresults)
assert (testresults == _testresults)
tt.tock("tested")
def run(traindomains="ALL",
domain="recipes",
mincoverage=2,
lr=0.001,
enclrmul=0.1,
numbeam=1,
ftlr=0.0001,
cosinelr=False,
warmup=0.,
batsize=30,
pretrainbatsize=100,
epochs=100,
resetmode="none",
pretrainepochs=100,
minpretrainepochs=10,
dropout=0.1,
decoderdropout=0.5,
wreg=1e-9,
gradnorm=3,
smoothing=0.,
patience=5,
gpu=-1,
seed=123456789,
encoder="bert-base-uncased",
numlayers=6,
hdim=600,
numheads=8,
maxlen=30,
localtest=False,
printtest=False,
fullsimplify=True,
nopretrain=False,
onlyabstract=False,
pretrainsetting="all", # "all", "all+lex", "lex"
finetunesetting="min", # "lex", "all", "min"
):
settings = locals().copy()
print(json.dumps(settings, indent=4))
numresets, resetafter, resetevery = 0, 0, 0
if resetmode == "none":
pass
elif resetmode == "once":
resetafter = 15
resetevery = 5
numresets = 1
elif resetmode == "more":
resetafter = 15
resetevery = 5
numresets = 3
elif resetmode == "forever":
resetafter = 15
resetevery = 5
numresets = 1000
print(f'Resetting: "{resetmode}": {numresets} times, first after {resetafter} epochs, then every {resetevery} epochs')
# wandb.init(project=f"overnight_joint_pretrain_fewshot_{pretrainsetting}-{finetunesetting}-{domain}",
# reinit=True, config=settings)
if traindomains == "ALL":
alldomains = {"recipes", "restaurants", "blocks", "calendar", "housing", "publications"}
traindomains = alldomains - {domain, }
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if gpu < 0 else torch.device(gpu)
tt.tick("loading data")
tds, ftds, vds, fvds, xds, nltok, flenc, generaltokenmask = \
load_ds(traindomains=traindomains, testdomain=domain, nl_mode=encoder, mincoverage=mincoverage,
fullsimplify=fullsimplify, onlyabstract=onlyabstract,
pretrainsetting=pretrainsetting, finetunesetting=finetunesetting)
tt.msg(f"{len(tds)/(len(tds) + len(vds)):.2f}/{len(vds)/(len(tds) + len(vds)):.2f} ({len(tds)}/{len(vds)}) train/valid")
tt.msg(f"{len(ftds)/(len(ftds) + len(fvds) + len(xds)):.2f}/{len(fvds)/(len(ftds) + len(fvds) + len(xds)):.2f}/{len(xds)/(len(ftds) + len(fvds) + len(xds)):.2f} ({len(ftds)}/{len(fvds)}/{len(xds)}) fttrain/ftvalid/test")
tdl = DataLoader(tds, batch_size=pretrainbatsize, shuffle=True, collate_fn=partial(autocollate, pad_value=0))
ftdl = DataLoader(ftds, batch_size=batsize, shuffle=True, collate_fn=partial(autocollate, pad_value=0))
vdl = DataLoader(vds, batch_size=pretrainbatsize, shuffle=False, collate_fn=partial(autocollate, pad_value=0))
fvdl = DataLoader(fvds, batch_size=batsize, shuffle=False, collate_fn=partial(autocollate, pad_value=0))
xdl = DataLoader(xds, batch_size=batsize, shuffle=False, collate_fn=partial(autocollate, pad_value=0))
tt.tock("data loaded")
tt.tick("creating model")
trainm, testm = create_model(encoder_name=encoder,
dec_vocabsize=flenc.vocab.number_of_ids(),
dec_layers=numlayers,
dec_dim=hdim,
dec_heads=numheads,
dropout=dropout,
decoderdropout=decoderdropout,
smoothing=smoothing,
maxlen=maxlen,
numbeam=numbeam,
tensor2tree=partial(_tensor2tree, D=flenc.vocab),
generaltokenmask=generaltokenmask,
resetmode=resetmode
)
tt.tock("model created")
# run a batch of data through the model
if localtest:
batch = next(iter(tdl))
out = trainm(*batch)
print(out)
out = testm(*batch)
print(out)
# region pretrain on all domains
metrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
vmetrics = make_array_of_metrics("seq_acc", "tree_acc")
xmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params if k not in exclude_params]
tt.msg("different param groups")
encparams = [v for k, v in trainable_params if k.startswith("model.model.encoder")]
otherparams = [v for k, v in trainable_params if not k.startswith("model.model.encoder")]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{"params": encparams, "lr": lr * enclrmul},
{"params": otherparams}]
optim = torch.optim.Adam(paramgroups, lr=lr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(trainm.parameters(), gradnorm)
if resetmode != "none":
minpretrainepochs = resetafter + (numresets - 1) * resetevery
eyt = q.EarlyStopper(vmetrics[1], patience=patience, min_epochs=minpretrainepochs,
more_is_better=True, remember_f=lambda: deepcopy(trainm.model))
reinit = Reinitializer(trainm.model, resetafter=resetafter, resetevery=resetevery, numresets=numresets, resetothers=[eyt])
# def wandb_logger():
# d = {}
# for name, loss in zip(["loss", "elem_acc", "seq_acc", "tree_acc"], metrics):
# d["train_"+name] = loss.get_epoch_error()
# for name, loss in zip(["seq_acc", "tree_acc"], vmetrics):
# d["valid_"+name] = loss.get_epoch_error()
# wandb.log(d)
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(steps=t_max-warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(optim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch, model=trainm, dataloader=tdl, optim=optim, losses=metrics,
_train_batch=trainbatch, device=device, on_end=[lambda: lr_schedule.step()])
validepoch = partial(q.test_epoch, model=testm, dataloader=vdl, losses=vmetrics, device=device,
on_end=[lambda: eyt.on_epoch_end(), lambda: reinit()])#, lambda: wandb_logger()])
if not nopretrain:
tt.tick("pretraining")
q.run_training(run_train_epoch=trainepoch, run_valid_epoch=validepoch, max_epochs=pretrainepochs,
check_stop=[lambda: eyt.check_stop()])
tt.tock("done pretraining")
if eyt.get_remembered() is not None:
tt.msg("reloaded")
trainm.model = eyt.get_remembered()
testm.model = eyt.get_remembered()
# endregion
# region finetune
ftmetrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
ftvmetrics = make_array_of_metrics("seq_acc", "tree_acc")
ftxmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params if k not in exclude_params]
tt.msg("different param groups")
encparams = [v for k, v in trainable_params if k.startswith("model.model.encoder")]
otherparams = [v for k, v in trainable_params if not k.startswith("model.model.encoder")]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{"params": encparams, "lr": ftlr * enclrmul},
{"params": otherparams}]
ftoptim = torch.optim.Adam(paramgroups, lr=ftlr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(trainm.parameters(), gradnorm)
# def wandb_logger_ft():
# d = {}
# for name, loss in zip(["loss", "elem_acc", "seq_acc", "tree_acc"], ftmetrics):
# d["ft_train_" + name] = loss.get_epoch_error()
# for name, loss in zip(["seq_acc", "tree_acc"], ftvmetrics):
# d["ft_valid_" + name] = loss.get_epoch_error()
# wandb.log(d)
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(steps=t_max - warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(ftoptim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch, model=trainm, dataloader=ftdl, optim=ftoptim, losses=ftmetrics,
_train_batch=trainbatch, device=device, on_end=[lambda: lr_schedule.step()])
validepoch = partial(q.test_epoch, model=testm, dataloader=fvdl, losses=ftvmetrics, device=device,
on_end=[])#, lambda: wandb_logger_ft()])
tt.tick("finetuning")
q.run_training(run_train_epoch=trainepoch, run_valid_epoch=validepoch, max_epochs=epochs)
tt.tock("done finetuning")
# endregion
tt.tick("testing")
validresults = q.test_epoch(model=testm, dataloader=fvdl, losses=ftvmetrics, device=device)
testresults = q.test_epoch(model=testm, dataloader=xdl, losses=ftxmetrics, device=device)
print(validresults)
print(testresults)
tt.tock("tested")
if printtest:
predm = testm.model
predm.to(device)
c, t = 0, 0
for testbatch in iter(xdl):
input_ids = testbatch[0]
output_ids = testbatch[1]
input_ids = input_ids.to(device)
ret = predm.generate(input_ids, attention_mask=input_ids != predm.config.pad_token_id,
max_length=maxlen)
inp_strs = [nltok.decode(input_idse, skip_special_tokens=True, clean_up_tokenization_spaces=False) for input_idse in input_ids]
out_strs = [flenc.vocab.tostr(rete.to(torch.device("cpu"))) for rete in ret]
gold_strs = [flenc.vocab.tostr(output_idse.to(torch.device("cpu"))) for output_idse in output_ids]
for x, y, g in zip(inp_strs, out_strs, gold_strs):
print(" ")
print(f"'{x}'\n--> {y}\n <=> {g}")
if y == g:
c += 1
else:
print("NOT SAME")
t += 1
print(f"seq acc: {c/t}")
# testout = q.eval_loop(model=testm, dataloader=xdl, device=device)
# print(testout)
print("done")
# settings.update({"train_seqacc": losses[]})
for metricarray, datasplit in zip([ftmetrics, ftvmetrics, ftxmetrics], ["train", "valid", "test"]):
for metric in metricarray:
settings[f"{datasplit}_{metric.name}"] = metric.get_epoch_error()
# wandb.config.update(settings)
# print(settings)
return settings
def run(lr=0.001,
batsize=20,
epochs=60,
embdim=128,
encdim=256,
numlayers=1,
beamsize=1,
dropout=.25,
wreg=1e-10,
cuda=False,
gpu=0,
minfreq=2,
gradnorm=3.,
smoothing=0.,
cosine_restarts=1.,
seed=456789,
p_step=.2,
p_min=.3,
):
localargs = locals().copy()
print(locals())
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if not cuda else torch.device("cuda", gpu)
tt.tick("loading data")
ds = GeoDataset(sentence_encoder=SequenceEncoder(tokenizer=split_tokenizer), min_freq=minfreq)
print(f"max lens: {ds.maxlen_input} (input) and {ds.maxlen_output} (output)")
tt.tock("data loaded")
do_rare_stats(ds)
# batch = next(iter(train_dl))
# print(batch)
# print("input graph")
# print(batch.batched_states)
model = BasicGenModel(embdim=embdim, hdim=encdim, dropout=dropout, numlayers=numlayers,
sentence_encoder=ds.sentence_encoder, query_encoder=ds.query_encoder, feedatt=True,
p_step=p_step, p_min=p_min)
# sentence_rare_tokens = set([ds.sentence_encoder.vocab(i) for i in model.inp_emb.rare_token_ids])
# do_rare_stats(ds, sentence_rare_tokens=sentence_rare_tokens)
losses = [CELoss(ignore_index=0, mode="logprobs", smoothing=smoothing)]
tfdecoder = SeqDecoder(model, tf_ratio=1.,
eval=losses + [SeqAccuracies(), TreeAccuracy(tensor2tree=partial(tensor2tree, D=ds.query_encoder.vocab),
orderless={"and", "or"})])
losses = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
# beamdecoder = BeamActionSeqDecoder(tfdecoder.model, beamsize=beamsize, maxsteps=50)
if beamsize == 1:
freedecoder = SeqDecoder(model, maxtime=100, tf_ratio=0.,
eval=[SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(tensor2tree, D=ds.query_encoder.vocab),
orderless={"and", "or"})])
vlosses = make_array_of_metrics("seq_acc", "tree_acc")
else:
freedecoder = BeamDecoder(model, maxtime=100, beamsize=beamsize,
eval=[SeqAccuracies()],
eval_beam=[TreeAccuracy(tensor2tree=partial(tensor2tree, D=ds.query_encoder.vocab),
orderless={"and", "or"})])
vlosses = make_array_of_metrics("seq_acc", "tree_acc", "tree_acc_at_last")
# 4. define optim
# optim = torch.optim.Adam(trainable_params, lr=lr, weight_decay=wreg)
optim = torch.optim.Adam(tfdecoder.parameters(), lr=lr, weight_decay=wreg)
# lr schedule
if cosine_restarts >= 0:
# t_max = epochs * len(train_dl)
t_max = epochs
print(f"Total number of updates: {t_max}")
lr_schedule = q.WarmupCosineWithHardRestartsSchedule(optim, 0, t_max, cycles=cosine_restarts)
reduce_lr = [lambda: lr_schedule.step()]
else:
reduce_lr = []
# 6. define training function
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(tfdecoder.parameters(), gradnorm)
# clipgradnorm = lambda: None
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch, model=tfdecoder, dataloader=ds.dataloader("train", batsize), optim=optim, losses=losses,
_train_batch=trainbatch, device=device, on_end=reduce_lr)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch, model=freedecoder, dataloader=ds.dataloader("test", batsize), losses=vlosses, device=device)
# validepoch = partial(q.test_epoch, model=freedecoder, dataloader=valid_dl, losses=vlosses, device=device)
# p = q.save_run(freedecoder, localargs, filepath=__file__)
# q.save_dataset(ds, p)
# _freedecoder, _localargs = q.load_run(p)
# _ds = q.load_dataset(p)
# sys.exit()
# 7. run training
tt.tick("training")
q.run_training(run_train_epoch=trainepoch, run_valid_epoch=validepoch, max_epochs=epochs)
tt.tock("done training")
# testing
tt.tick("testing")
testresults = q.test_epoch(model=freedecoder, dataloader=ds.dataloader("test", batsize), losses=vlosses, device=device)
print("validation test results: ", testresults)
tt.tock("tested")
tt.tick("testing")
testresults = q.test_epoch(model=freedecoder, dataloader=ds.dataloader("test", batsize), losses=vlosses, device=device)
print("test results: ", testresults)
tt.tock("tested")
# save model?
tosave = input("Save this model? 'y(es)'=Yes, <int>=overwrite previous, otherwise=No) \n>")
if tosave.lower() == "y" or tosave.lower() == "yes" or re.match("\d+", tosave.lower()):
overwrite = int(tosave) if re.match("\d+", tosave) else None
p = q.save_run(model, localargs, filepath=__file__, overwrite=overwrite)
q.save_dataset(ds, p)
_model, _localargs = q.load_run(p)
_ds = q.load_dataset(p)
_freedecoder = BeamDecoder(_model, maxtime=50, beamsize=beamsize,
eval_beam=[TreeAccuracy(tensor2tree=partial(tensor2tree, D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})])
# testing
tt.tick("testing reloaded")
_testresults = q.test_epoch(model=_freedecoder, dataloader=_ds.dataloader("test", batsize),
losses=vlosses, device=device)
print(_testresults)
assert(testresults == _testresults)
tt.tock("tested")
def run(
lr=2.5e-4,
edropout=0.1,
wdropout=0.1,
rdropout=0.1,
adropout=0.1,
dropout=-1.,
numlayers=2,
numheads=8,
abspos=False,
tie_wordvecs=False,
gradnorm=0.5,
epochs=200,
dim=256,
seqlen=50,
batsize=32,
eval_batsize=64,
cuda=False,
gpu=0,
test=True,
subsampleeval=10,
wreg=1e-6,
lrcycle=5,
lrwarmup=3,
):
tt = q.ticktock("script")
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
tt.tick("loading data")
train_batches, valid_batches, test_batches, D = \
load_data(batsize=batsize, eval_batsize=eval_batsize, seqlen=seqlen, subsample_eval=subsampleeval)
tt.tock("data loaded")
print("{} batches in train".format(len(train_batches)))
if dropout >= 0.:
edropout, adropout, rdropout, wdropout = dropout, dropout, dropout, dropout
relpos = not abspos
tt.tick("creating model")
m = TransformerLM(dim=dim,
worddic=D,
numlayers=numlayers,
numheads=numheads,
activation=q.GeLU,
embedding_dropout=edropout,
attention_dropout=adropout,
word_dropout=wdropout,
residual_dropout=rdropout,
relpos=relpos,
tie_wordvecs=tie_wordvecs,
maxlen=2 * seqlen).to(device)
valid_m = TransformerLMCell(m)
if test:
for i, batch in enumerate(valid_batches):
batch = [batch_e.to(device) for batch_e in batch]
y = valid_m(batch[0])
if i > 5:
break
for i, batch in enumerate(valid_batches):
pass
print(i, batsize, seqlen, valid_batches.data.size(0))
print(y.size())
# return
# return
loss = q.LossWrapper(q.CELoss(mode="logits"))
validloss = q.LossWrapper(q.CELoss(mode="logits"))
validlosses = [validloss, PPLfromCE(validloss)]
testloss = q.LossWrapper(q.CELoss(mode="logits"))
testlosses = [testloss, PPLfromCE(testloss)]
for l in [loss] + validlosses + testlosses: # put losses on right device
l.loss.to(device)
# optim = torch.optim.SGD(m.parameters(), lr=lr)
numbats = len(train_batches)
print("{} batches in training".format(numbats))
optim = torch.optim.Adam(m.parameters(), lr=lr, weight_decay=wreg)
# lrp = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, mode="min", factor=1/4, patience=0, verbose=True)
# lrp_f = lambda: lrp.step(validloss.get_epoch_error())
sched = q.CosineLRwithWarmup(optim,
lrcycle * numbats,
warmup=lrwarmup * numbats)
train_batch_f = partial(
q.train_batch,
on_before_optim_step=[
lambda: torch.nn.utils.clip_grad_norm_(m.parameters(), gradnorm),
lambda: sched.step()
])
train_epoch_f = partial(q.train_epoch,
model=m,
dataloader=train_batches,
optim=optim,
losses=[loss],
device=device,
_train_batch=train_batch_f)
valid_epoch_f = partial(q.test_epoch,
model=valid_m,
dataloader=valid_batches,
losses=validlosses,
device=device)
tt.tock("created model")
tt.tick("training")
q.run_training(train_epoch_f,
valid_epoch_f,
max_epochs=epochs,
validinter=1)
tt.tock("trained")
tt.tick("testing")
testresults = q.test_epoch(model=valid_m,
dataloader=test_batches,
losses=testlosses,
device=device)
print(testresults)
tt.tock("tested")
def run(
lr=0.001,
batsize=20,
epochs=60,
embdim=128,
encdim=256,
numlayers=1,
beamsize=5,
dropout=.25,
wreg=1e-10,
cuda=False,
gpu=0,
minfreq=2,
gradnorm=3.,
smoothing=0.1,
cosine_restarts=1.,
seed=123456,
numcvfolds=6,
testfold=-1, # if non-default, must be within number of splits, the chosen value is used for validation
reorder_random=False,
):
localargs = locals().copy()
print(locals())
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if not cuda else torch.device("cuda", gpu)
tt.tick("loading data")
cvfolds = None if testfold == -1 else numcvfolds
testfold = None if testfold == -1 else testfold
ds = GeoDataset(
sentence_encoder=SequenceEncoder(tokenizer=split_tokenizer),
min_freq=minfreq,
cvfolds=cvfolds,
testfold=testfold,
reorder_random=reorder_random)
print(
f"max lens: {ds.maxlen_input} (input) and {ds.maxlen_output} (output)")
tt.tock("data loaded")
do_rare_stats(ds)
# batch = next(iter(train_dl))
# print(batch)
# print("input graph")
# print(batch.batched_states)
model = BasicGenModel(embdim=embdim,
hdim=encdim,
dropout=dropout,
numlayers=numlayers,
sentence_encoder=ds.sentence_encoder,
query_encoder=ds.query_encoder,
feedatt=True)
# sentence_rare_tokens = set([ds.sentence_encoder.vocab(i) for i in model.inp_emb.rare_token_ids])
# do_rare_stats(ds, sentence_rare_tokens=sentence_rare_tokens)
tfdecoder = SeqDecoder(model,
tf_ratio=1.,
eval=[
CELoss(ignore_index=0,
mode="logprobs",
smoothing=smoothing),
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(
tensor2tree, D=ds.query_encoder.vocab),
orderless={"and"})
])
losses = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
freedecoder = SeqDecoder(model,
maxtime=100,
tf_ratio=0.,
eval=[
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(
tensor2tree, D=ds.query_encoder.vocab),
orderless={"and"})
])
vlosses = make_array_of_metrics("seq_acc", "tree_acc")
beamdecoder = BeamDecoder(model,
maxtime=100,
beamsize=beamsize,
copy_deep=True,
eval=[SeqAccuracies()],
eval_beam=[
TreeAccuracy(tensor2tree=partial(
tensor2tree, D=ds.query_encoder.vocab),
orderless={"and"})
])
beamlosses = make_array_of_metrics("seq_acc", "tree_acc",
"tree_acc_at_last")
# 4. define optim
# optim = torch.optim.Adam(trainable_params, lr=lr, weight_decay=wreg)
optim = torch.optim.Adam(tfdecoder.parameters(), lr=lr, weight_decay=wreg)
# lr schedule
if cosine_restarts >= 0:
# t_max = epochs * len(train_dl)
t_max = epochs
print(f"Total number of updates: {t_max}")
lr_schedule = q.WarmupCosineWithHardRestartsSchedule(
optim, 0, t_max, cycles=cosine_restarts)
reduce_lr = [lambda: lr_schedule.step()]
else:
reduce_lr = []
# 6. define training function
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
tfdecoder.parameters(), gradnorm)
# clipgradnorm = lambda: None
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
train_on = "train"
valid_on = "test" if testfold is None else "valid"
trainepoch = partial(q.train_epoch,
model=tfdecoder,
dataloader=ds.dataloader(train_on,
batsize,
shuffle=True),
optim=optim,
losses=losses,
_train_batch=trainbatch,
device=device,
on_end=reduce_lr)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=freedecoder,
dataloader=ds.dataloader(valid_on,
batsize,
shuffle=False),
losses=vlosses,
device=device)
# validepoch = partial(q.test_epoch, model=freedecoder, dataloader=valid_dl, losses=vlosses, device=device)
# p = q.save_run(freedecoder, localargs, filepath=__file__)
# q.save_dataset(ds, p)
# _freedecoder, _localargs = q.load_run(p)
# _ds = q.load_dataset(p)
# sys.exit()
# 7. run training
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs)
tt.tock("done training")
if testfold is not None:
return vlosses[1].get_epoch_error()
# testing
tt.tick("testing")
testresults = q.test_epoch(model=beamdecoder,
dataloader=ds.dataloader("test", batsize),
losses=beamlosses,
device=device)
print("validation test results: ", testresults)
tt.tock("tested")
tt.tick("testing")
testresults = q.test_epoch(model=beamdecoder,
dataloader=ds.dataloader("test", batsize),
losses=beamlosses,
device=device)
print("test results: ", testresults)
tt.tock("tested")
# save model?
tosave = input(
"Save this model? 'y(es)'=Yes, <int>=overwrite previous, otherwise=No) \n>"
)
# if True:
# overwrite = None
if tosave.lower() == "y" or tosave.lower() == "yes" or re.match(
"\d+", tosave.lower()):
overwrite = int(tosave) if re.match("\d+", tosave) else None
p = q.save_run(model,
localargs,
filepath=__file__,
overwrite=overwrite)
q.save_dataset(ds, p)
_model, _localargs = q.load_run(p)
_ds = q.load_dataset(p)
_freedecoder = BeamDecoder(_model,
maxtime=100,
beamsize=beamsize,
copy_deep=True,
eval=[SeqAccuracies()],
eval_beam=[
TreeAccuracy(tensor2tree=partial(
tensor2tree,
D=ds.query_encoder.vocab),
orderless={"and"})
])
# testing
tt.tick("testing reloaded")
_testresults = q.test_epoch(model=_freedecoder,
dataloader=_ds.dataloader("test", batsize),
losses=beamlosses,
device=device)
print(_testresults)
tt.tock("tested")
# save predictions
_, testpreds = q.eval_loop(_freedecoder,
ds.dataloader("test",
batsize=batsize,
shuffle=False),
device=device)
testout = get_outputs_for_save(testpreds)
_, trainpreds = q.eval_loop(_freedecoder,
ds.dataloader("train",
batsize=batsize,
shuffle=False),
device=device)
trainout = get_outputs_for_save(trainpreds)
with open(os.path.join(p, "trainpreds.json"), "w") as f:
ujson.dump(trainout, f)
with open(os.path.join(p, "testpreds.json"), "w") as f:
ujson.dump(testout, f)
def run(
domain="restaurants",
lr=0.001,
enclrmul=0.1,
cosinelr=False,
warmup=0.,
batsize=20,
epochs=100,
dropout=0.1,
wreg=1e-9,
gradnorm=3,
smoothing=0.,
patience=5,
gpu=-1,
seed=123456789,
encoder="bart-large",
numlayers=6,
hdim=600,
numheads=8,
maxlen=50,
localtest=False,
printtest=False,
trainonvalid=False,
):
settings = locals().copy()
print(locals())
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if gpu < 0 else torch.device(gpu)
tt.tick("loading data")
tds, vds, xds, nltok, flenc = load_ds(domain=domain,
nl_mode=encoder,
trainonvalid=trainonvalid)
tdl = DataLoader(tds,
batch_size=batsize,
shuffle=True,
collate_fn=partial(autocollate, pad_value=1))
vdl = DataLoader(vds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(autocollate, pad_value=1))
xdl = DataLoader(xds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(autocollate, pad_value=1))
tt.tock("data loaded")
tt.tick("creating model")
trainm, testm = create_model(encoder_name=encoder,
dec_vocabsize=flenc.vocab.number_of_ids(),
dec_layers=numlayers,
dec_dim=hdim,
dec_heads=numheads,
dropout=dropout,
smoothing=smoothing,
maxlen=maxlen,
tensor2tree=partial(_tensor2tree,
D=flenc.vocab))
tt.tock("model created")
# run a batch of data through the model
if localtest:
batch = next(iter(tdl))
out = trainm(*batch)
print(out)
out = testm(*batch)
print(out)
metrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
vmetrics = make_array_of_metrics("seq_acc", "tree_acc")
xmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params
if k not in exclude_params]
tt.msg("different param groups")
encparams = [
v for k, v in trainable_params if k.startswith("model.model.encoder")
]
otherparams = [
v for k, v in trainable_params
if not k.startswith("model.model.encoder")
]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{
"params": encparams,
"lr": lr * enclrmul
}, {
"params": otherparams
}]
optim = torch.optim.Adam(paramgroups, lr=lr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
trainm.parameters(), gradnorm)
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(
steps=t_max - warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(optim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=trainm,
dataloader=tdl,
optim=optim,
losses=metrics,
_train_batch=trainbatch,
device=device,
on_end=[lambda: lr_schedule.step()])
validepoch = partial(q.test_epoch,
model=testm,
dataloader=vdl,
losses=vmetrics,
device=device)
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs)
tt.tock("done training")
tt.tick("testing")
validresults = q.test_epoch(model=testm,
dataloader=vdl,
losses=vmetrics,
device=device)
testresults = q.test_epoch(model=testm,
dataloader=xdl,
losses=xmetrics,
device=device)
print(validresults)
print(testresults)
tt.tock("tested")
if printtest:
predm = testm.model
predm.to(device)
c, t = 0, 0
for testbatch in iter(xdl):
input_ids = testbatch[0]
output_ids = testbatch[1]
input_ids = input_ids.to(device)
ret = predm.generate(
input_ids,
attention_mask=input_ids != predm.config.pad_token_id,
max_length=maxlen)
inp_strs = [
nltok.decode(input_idse,
skip_special_tokens=True,
clean_up_tokenization_spaces=False)
for input_idse in input_ids
]
out_strs = [
flenc.vocab.tostr(rete.to(torch.device("cpu"))) for rete in ret
]
gold_strs = [
flenc.vocab.tostr(output_idse.to(torch.device("cpu")))
for output_idse in output_ids
]
for x, y, g in zip(inp_strs, out_strs, gold_strs):
print(" ")
print(f"'{x}'\n--> {y}\n <=> {g}")
if y == g:
c += 1
else:
print("NOT SAME")
t += 1
print(f"seq acc: {c/t}")
# testout = q.eval_loop(model=testm, dataloader=xdl, device=device)
# print(testout)
print("done")
# settings.update({"train_seqacc": losses[]})
for metricarray, datasplit in zip([metrics, vmetrics, xmetrics],
["train", "valid", "test"]):
for metric in metricarray:
settings[f"{datasplit}_{metric.name}"] = metric.get_epoch_error()
# print(settings)
return settings
def run_rerank(
lr=0.001,
batsize=20,
epochs=1,
embdim=301, # not used
encdim=200,
numlayers=1,
beamsize=5,
dropout=.2,
wreg=1e-10,
cuda=False,
gpu=0,
minfreq=2,
gradnorm=3.,
cosine_restarts=1.,
domain="restaurants",
gensavedp="overnight_basic/run{}",
genrunid=1,
):
localargs = locals().copy()
print(locals())
gensavedrunp = gensavedp.format(genrunid)
tt = q.ticktock("script")
device = torch.device("cpu") if not cuda else torch.device("cuda", gpu)
tt.tick("loading data")
ds = q.load_dataset(gensavedrunp)
# ds = OvernightDataset(domain=domain, sentence_encoder=SequenceEncoder(tokenizer=split_tokenizer), min_freq=minfreq)
print(
f"max lens: {ds.maxlen_input} (input) and {ds.maxlen_output} (output)")
tt.tock("data loaded")
do_rare_stats(ds)
# batch = next(iter(train_dl))
# print(batch)
# print("input graph")
# print(batch.batched_states)
genmodel, genargs = q.load_run(gensavedrunp)
# BasicGenModel(embdim=embdim, hdim=encdim, dropout=dropout, numlayers=numlayers,
# sentence_encoder=ds.sentence_encoder, query_encoder=ds.query_encoder, feedatt=True)
# sentence_rare_tokens = set([ds.sentence_encoder.vocab(i) for i in model.inp_emb.rare_token_ids])
# do_rare_stats(ds, sentence_rare_tokens=sentence_rare_tokens)
inpenc = q.LSTMEncoder(embdim,
*([encdim // 2] * numlayers),
bidir=True,
dropout_in=dropout)
outenc = q.LSTMEncoder(embdim,
*([encdim // 2] * numlayers),
bidir=True,
dropout_in=dropout)
scoremodel = SimpleScoreModel(genmodel.inp_emb, genmodel.out_emb,
LSTMEncoderWrapper(inpenc),
LSTMEncoderWrapper(outenc), DotSimilarity())
model = BeamReranker(genmodel, scoremodel, beamsize=beamsize, maxtime=50)
# todo: run over whole dataset to populate beam cache
testbatch = next(iter(ds.dataloader("train", batsize=2)))
model(testbatch)
sys.exit()
tfdecoder = SeqDecoder(TFTransition(model), [
CELoss(ignore_index=0, mode="logprobs"),
SeqAccuracies(),
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})
])
# beamdecoder = BeamActionSeqDecoder(tfdecoder.model, beamsize=beamsize, maxsteps=50)
freedecoder = BeamDecoder(
model,
maxtime=50,
beamsize=beamsize,
eval_beam=[
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})
])
# # test
# tt.tick("doing one epoch")
# for batch in iter(train_dl):
# batch = batch.to(device)
# ttt.tick("start batch")
# # with torch.no_grad():
# out = tfdecoder(batch)
# ttt.tock("end batch")
# tt.tock("done one epoch")
# print(out)
# sys.exit()
# beamdecoder(next(iter(train_dl)))
# print(dict(tfdecoder.named_parameters()).keys())
losses = make_array_of_metrics("loss", "seq_acc", "tree_acc")
vlosses = make_array_of_metrics("tree_acc", "tree_acc_at3",
"tree_acc_at_last")
trainable_params = tfdecoder.named_parameters()
exclude_params = {"model.model.inp_emb.emb.weight"
} # don't train input embeddings if doing glove
trainable_params = [
v for k, v in trainable_params if k not in exclude_params
]
# 4. define optim
optim = torch.optim.Adam(trainable_params, lr=lr, weight_decay=wreg)
# optim = torch.optim.SGD(tfdecoder.parameters(), lr=lr, weight_decay=wreg)
# lr schedule
if cosine_restarts >= 0:
# t_max = epochs * len(train_dl)
t_max = epochs
print(f"Total number of updates: {t_max}")
lr_schedule = q.WarmupCosineWithHardRestartsSchedule(
optim, 0, t_max, cycles=cosine_restarts)
reduce_lr = [lambda: lr_schedule.step()]
else:
reduce_lr = []
# 6. define training function
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
tfdecoder.parameters(), gradnorm)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=tfdecoder,
dataloader=ds.dataloader("train", batsize),
optim=optim,
losses=losses,
_train_batch=trainbatch,
device=device,
on_end=reduce_lr)
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=freedecoder,
dataloader=ds.dataloader("valid", batsize),
losses=vlosses,
device=device)
# validepoch = partial(q.test_epoch, model=freedecoder, dataloader=valid_dl, losses=vlosses, device=device)
# p = q.save_run(freedecoder, localargs, filepath=__file__)
# q.save_dataset(ds, p)
# _freedecoder, _localargs = q.load_run(p)
# _ds = q.load_dataset(p)
# sys.exit()
# 7. run training
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs)
tt.tock("done training")
# testing
tt.tick("testing")
testresults = q.test_epoch(model=freedecoder,
dataloader=ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print(testresults)
tt.tock("tested")
# save model?
tosave = input(
"Save this model? 'y(es)'=Yes, <int>=overwrite previous, otherwise=No) \n>"
)
if tosave.lower() == "y" or tosave.lower() == "yes" or re.match(
"\d+", tosave.lower()):
overwrite = int(tosave) if re.match("\d+", tosave) else None
p = q.save_run(model,
localargs,
filepath=__file__,
overwrite=overwrite)
q.save_dataset(ds, p)
_model, _localargs = q.load_run(p)
_ds = q.load_dataset(p)
_freedecoder = BeamDecoder(
_model,
maxtime=50,
beamsize=beamsize,
eval_beam=[
TreeAccuracy(tensor2tree=partial(tensor2tree,
D=ds.query_encoder.vocab),
orderless={"op:and", "SW:concat"})
])
# testing
tt.tick("testing reloaded")
_testresults = q.test_epoch(model=_freedecoder,
dataloader=_ds.dataloader("test", batsize),
losses=vlosses,
device=device)
print(_testresults)
assert (testresults == _testresults)
tt.tock("tested")
def run(
sourcelang="en",
supportlang="en",
testlang="en",
lr=0.001,
enclrmul=0.1,
numbeam=1,
cosinelr=False,
warmup=0.,
batsize=20,
epochs=100,
dropout=0.1,
dropoutdec=0.1,
wreg=1e-9,
gradnorm=3,
smoothing=0.,
patience=5,
gpu=-1,
seed=123456789,
encoder="xlm-roberta-base",
numlayers=6,
hdim=600,
numheads=8,
maxlen=50,
localtest=False,
printtest=False,
trainonvalid=False,
statesimweight=0.,
probsimweight=0.,
projmode="simple", # "simple" or "twolayer"
):
settings = locals().copy()
print(json.dumps(settings, indent=4))
# wandb.init(project=f"overnight_pretrain_bert-{domain}",
# reinit=True, config=settings)
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if gpu < 0 else torch.device(gpu)
tt.tick("loading data")
nltok_name = encoder
tds, vds, xds, nltok, flenc = load_multilingual_geoquery(
sourcelang,
supportlang,
testlang,
nltok_name=nltok_name,
trainonvalid=trainonvalid)
tt.msg(
f"{len(tds)/(len(tds) + len(vds) + len(xds)):.2f}/{len(vds)/(len(tds) + len(vds) + len(xds)):.2f}/{len(xds)/(len(tds) + len(vds) + len(xds)):.2f} ({len(tds)}/{len(vds)}/{len(xds)}) train/valid/test"
)
tdl = DataLoader(tds,
batch_size=batsize,
shuffle=True,
collate_fn=partial(collate_fn,
pad_value_nl=nltok.pad_token_id))
vdl = DataLoader(vds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(collate_fn,
pad_value_nl=nltok.pad_token_id))
xdl = DataLoader(xds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(collate_fn,
pad_value_nl=nltok.pad_token_id))
tt.tock("data loaded")
tt.tick("creating model")
trainm, testm = create_model(
encoder_name=encoder,
dec_vocabsize=flenc.vocab.number_of_ids(),
dec_layers=numlayers,
dec_dim=hdim,
dec_heads=numheads,
dropout=dropout,
dropoutdec=dropoutdec,
smoothing=smoothing,
maxlen=maxlen,
numbeam=numbeam,
tensor2tree=partial(_tensor2tree, D=flenc.vocab),
statesimweight=statesimweight,
probsimweight=probsimweight,
projmode=projmode,
)
tt.tock("model created")
# run a batch of data through the model
if localtest:
batch = next(iter(tdl))
out = trainm(*batch)
print(out)
out = testm(*batch)
print(out)
metrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
vmetrics = make_array_of_metrics("seq_acc", "tree_acc")
xmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params
if k not in exclude_params]
tt.msg("different param groups")
encparams = [
v for k, v in trainable_params
if k.startswith("model.model.encoder.model")
]
otherparams = [
v for k, v in trainable_params
if not k.startswith("model.model.encoder.model")
]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{
"params": encparams,
"lr": lr * enclrmul
}, {
"params": otherparams
}]
optim = torch.optim.Adam(paramgroups, lr=lr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
trainm.parameters(), gradnorm)
eyt = q.EarlyStopper(vmetrics[-1],
patience=patience,
min_epochs=10,
more_is_better=True,
remember_f=lambda:
(deepcopy(trainm.model), deepcopy(trainm.model2)))
# def wandb_logger():
# d = {}
# for name, loss in zip(["loss", "elem_acc", "seq_acc", "tree_acc"], metrics):
# d["_train_"+name] = loss.get_epoch_error()
# for name, loss in zip(["seq_acc", "tree_acc"], vmetrics):
# d["_valid_"+name] = loss.get_epoch_error()
# wandb.log(d)
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(
steps=t_max - warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(optim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=trainm,
dataloader=tdl,
optim=optim,
losses=metrics,
_train_batch=trainbatch,
device=device,
on_end=[lambda: lr_schedule.step()])
validepoch = partial(q.test_epoch,
model=testm,
dataloader=vdl,
losses=vmetrics,
device=device,
on_end=[lambda: eyt.on_epoch_end()
]) #, on_end=[lambda: wandb_logger()])
# validepoch() # TODO comment out after debugging
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs,
check_stop=[lambda: eyt.check_stop()])
tt.tock("done training")
if eyt.remembered is not None:
trainm.model = eyt.remembered[0]
trainm.model2 = eyt.remembered[1]
testm.model = eyt.remembered[0]
testm.model2 = eyt.remembered[1]
tt.msg("reloaded best")
tt.tick("testing")
validresults = q.test_epoch(model=testm,
dataloader=vdl,
losses=vmetrics,
device=device)
testresults = q.test_epoch(model=testm,
dataloader=xdl,
losses=xmetrics,
device=device)
print(validresults)
print(testresults)
tt.tock("tested")
if printtest:
predm = testm.model2
predm.to(device)
c, t = 0, 0
for testbatch in iter(xdl):
input_ids = testbatch[0]
output_ids = testbatch[1]
input_ids = input_ids.to(device)
ret = predm.generate(
input_ids,
attention_mask=input_ids != predm.config.pad_token_id,
max_length=maxlen)
inp_strs = [
nltok.decode(input_idse,
skip_special_tokens=True,
clean_up_tokenization_spaces=False)
for input_idse in input_ids
]
out_strs = [
flenc.vocab.tostr(rete.to(torch.device("cpu"))) for rete in ret
]
gold_strs = [
flenc.vocab.tostr(output_idse.to(torch.device("cpu")))
for output_idse in output_ids
]
for x, y, g in zip(inp_strs, out_strs, gold_strs):
print(" ")
print(f"'{x}'\n--> {y}\n <=> {g}")
if y == g:
c += 1
else:
print("NOT SAME")
t += 1
print(f"seq acc: {c/t}")
# testout = q.eval_loop(model=testm, dataloader=xdl, device=device)
# print(testout)
print("done")
# settings.update({"train_seqacc": losses[]})
for metricarray, datasplit in zip([metrics, vmetrics, xmetrics],
["train", "valid", "test"]):
for metric in metricarray:
settings[f"{datasplit}_{metric.name}"] = metric.get_epoch_error()
# wandb.config.update(settings)
# print(settings)
return settings
def run(
lr=20.,
dropout=0.2,
dropconnect=0.2,
gradnorm=0.25,
epochs=25,
embdim=200,
encdim=200,
numlayers=2,
tieweights=False,
distill="glove", # "rnnlm", "glove"
seqlen=35,
batsize=20,
eval_batsize=80,
cuda=False,
gpu=0,
test=False,
repretrain=False, # retrain base model instead of loading it
savepath="rnnlm.base.pt", # where to save after training
glovepath="../../../data/glove/glove.300d"):
tt = q.ticktock("script")
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
tt.tick("loading data")
train_batches, valid_batches, test_batches, D = \
load_data(batsize=batsize, eval_batsize=eval_batsize,
seqlen=VariableSeqlen(minimum=5, maximum_offset=10, mu=seqlen, sigma=0))
tt.tock("data loaded")
print("{} batches in train".format(len(train_batches)))
# region base training
loss = q.LossWrapper(q.CELoss(mode="logits"))
validloss = q.LossWrapper(q.CELoss(mode="logits"))
validlosses = [validloss, PPLfromCE(validloss)]
testloss = q.LossWrapper(q.CELoss(mode="logits"))
testlosses = [testloss, PPLfromCE(testloss)]
for l in [loss] + validlosses + testlosses: # put losses on right device
l.loss.to(device)
if os.path.exists(savepath) and repretrain is False:
tt.tick("reloading base model")
with open(savepath, "rb") as f:
m = torch.load(f)
m.to(device)
tt.tock("reloaded base model")
else:
tt.tick("preparing training base")
dims = [embdim] + ([encdim] * numlayers)
m = RNNLayer_LM(*dims,
worddic=D,
dropout=dropout,
tieweights=tieweights).to(device)
if test:
for i, batch in enumerate(train_batches):
y = m(batch[0])
if i > 5:
break
print(y.size())
optim = torch.optim.SGD(m.parameters(), lr=lr)
train_batch_f = partial(q.train_batch,
on_before_optim_step=[
lambda: torch.nn.utils.clip_grad_norm_(
m.parameters(), gradnorm)
])
lrp = torch.optim.lr_scheduler.ReduceLROnPlateau(optim,
mode="min",
factor=1 / 4,
patience=0,
verbose=True)
lrp_f = lambda: lrp.step(validloss.get_epoch_error())
train_epoch_f = partial(q.train_epoch,
model=m,
dataloader=train_batches,
optim=optim,
losses=[loss],
device=device,
_train_batch=train_batch_f)
valid_epoch_f = partial(q.test_epoch,
model=m,
dataloader=valid_batches,
losses=validlosses,
device=device,
on_end=[lrp_f])
tt.tock("prepared training base")
tt.tick("training base model")
q.run_training(train_epoch_f,
valid_epoch_f,
max_epochs=epochs,
validinter=1)
tt.tock("trained base model")
with open(savepath, "wb") as f:
torch.save(m, f)
tt.tick("testing base model")
testresults = q.test_epoch(model=m,
dataloader=test_batches,
losses=testlosses,
device=device)
print(testresults)
tt.tock("tested base model")
# endregion
# region distillation
tt.tick("preparing training student")
dims = [embdim] + ([encdim] * numlayers)
ms = RNNLayer_LM(*dims, worddic=D, dropout=dropout,
tieweights=tieweights).to(device)
loss = q.LossWrapper(q.DistillLoss(temperature=2.))
validloss = q.LossWrapper(q.CELoss(mode="logits"))
validlosses = [validloss, PPLfromCE(validloss)]
testloss = q.LossWrapper(q.CELoss(mode="logits"))
testlosses = [testloss, PPLfromCE(testloss)]
for l in [loss] + validlosses + testlosses: # put losses on right device
l.loss.to(device)
optim = torch.optim.SGD(ms.parameters(), lr=lr)
train_batch_f = partial(
train_batch_distill,
on_before_optim_step=[
lambda: torch.nn.utils.clip_grad_norm_(ms.parameters(), gradnorm)
])
lrp = torch.optim.lr_scheduler.ReduceLROnPlateau(optim,
mode="min",
factor=1 / 4,
patience=0,
verbose=True)
lrp_f = lambda: lrp.step(validloss.get_epoch_error())
if distill == "rnnlm":
mbase = m
goldgetter = None
elif distill == "glove":
mbase = None
tt.tick("creating gold getter based on glove")
goldgetter = GloveGoldGetter(glovepath, worddic=D)
goldgetter.to(device)
tt.tock("created gold getter")
else:
raise q.SumTingWongException("unknown distill mode {}".format(distill))
train_epoch_f = partial(train_epoch_distill,
model=ms,
dataloader=train_batches,
optim=optim,
losses=[loss],
device=device,
_train_batch=train_batch_f,
mbase=mbase,
goldgetter=goldgetter)
valid_epoch_f = partial(q.test_epoch,
model=ms,
dataloader=valid_batches,
losses=validlosses,
device=device,
on_end=[lrp_f])
tt.tock("prepared training student")
tt.tick("training student model")
q.run_training(train_epoch_f,
valid_epoch_f,
max_epochs=epochs,
validinter=1)
tt.tock("trained student model")
tt.tick("testing student model")
testresults = q.test_epoch(model=ms,
dataloader=test_batches,
losses=testlosses,
device=device)
print(testresults)
tt.tock("tested student model")
def example_usage_full_with_penalty_and_hyperparam():
# 1. define model
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.lin = torch.nn.Linear(5, 5)
self.sm = torch.nn.Softmax(-1)
self._pen = 0
def batch_reset(self): # called before every batch
self._pen = 0 # resets penalty
def get_penalty(
self): # must be specified to be called by PenaltyGetter
return self._pen
def forward(self, _x):
_y = self.lin(_x)
self._pen = torch.sum(_y, dim=1)
return self.sm(_y)
model = Model()
# 2. define data
x = torch.rand(100, 5)
y = torch.randint(0, 5, (100, ))
dataset = torch.utils.data.TensorDataset(x, y)
traindataset, validdataset, testdataset = torch.utils.data.random_split(
dataset, [70, 10, 20])
trainloader = torch.utils.data.DataLoader(traindataset,
batch_size=2,
shuffle=True)
validloader = torch.utils.data.DataLoader(validdataset,
batch_size=2,
shuffle=False)
testloader = torch.utils.data.DataLoader(testdataset,
batch_size=2,
shuffle=False)
# 3. define losses and penalties and wrap them
loss = torch.nn.CrossEntropyLoss(reduction="mean")
loss2 = torch.nn.CrossEntropyLoss(reduction="sum")
penweight = q.hyperparam(1.)
pen = q.PenaltyGetter(model, "get_penalty", factor=penweight)
loss = q.LossWrapper(loss)
loss2 = q.LossWrapper(loss2)
pen = q.LossWrapper(pen)
# 4. define optim
optim = torch.optim.SGD(model.parameters(), lr=1.)
# 5. other options (device, ...)
device = torch.device("cpu")
def on_start_train_epoch():
penweight.v /= 1.2
print(q.v(penweight))
# 6. define training function (using partial)
trainepoch = partial(q.train_epoch,
model=model,
dataloader=trainloader,
optim=optim,
losses=[loss, loss2, pen],
device=device,
on_start=[on_start_train_epoch])
# 7. define validation function (using partial)
validepoch = partial(q.test_epoch,
model=model,
dataloader=validloader,
losses=[loss, loss2],
device=device)
# 8. run training
run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=50)
# 9. run test function
testresults = q.test_epoch(model=model,
dataloader=testloader,
losses=[loss, loss2],
device=device)
print(testresults)
def run(
traindomains="ALL",
domain="recipes",
mincoverage=2,
lr=0.001,
advlr=-1,
enclrmul=0.1,
numbeam=1,
ftlr=0.0001,
cosinelr=False,
warmup=0.,
batsize=30,
epochs=100,
pretrainepochs=100,
dropout=0.1,
wreg=1e-9,
gradnorm=3,
smoothing=0.,
patience=5,
gpu=-1,
seed=123456789,
encoder="bert-base-uncased",
numlayers=6,
hdim=600,
numheads=8,
maxlen=30,
localtest=False,
printtest=False,
fullsimplify=True,
domainstart=False,
useall=False,
nopretrain=False,
entropycontrib=1.,
advsteps=5,
):
settings = locals().copy()
print(json.dumps(settings, indent=4))
if advlr < 0:
advlr = lr
if traindomains == "ALL":
alldomains = {
"recipes", "restaurants", "blocks", "calendar", "housing",
"publications"
}
traindomains = alldomains - {
domain,
}
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if gpu < 0 else torch.device(gpu)
tt.tick("loading data")
tds, ftds, vds, fvds, xds, nltok, flenc, absflenc = \
load_ds(traindomains=traindomains, testdomain=domain, nl_mode=encoder, mincoverage=mincoverage,
fullsimplify=fullsimplify, add_domain_start=domainstart, useall=useall)
advds = Dataset(tds.examples)
tt.msg(
f"{len(tds)/(len(tds) + len(vds)):.2f}/{len(vds)/(len(tds) + len(vds)):.2f} ({len(tds)}/{len(vds)}) train/valid"
)
tt.msg(
f"{len(ftds)/(len(ftds) + len(fvds) + len(xds)):.2f}/{len(fvds)/(len(ftds) + len(fvds) + len(xds)):.2f}/{len(xds)/(len(ftds) + len(fvds) + len(xds)):.2f} ({len(ftds)}/{len(fvds)}/{len(xds)}) fttrain/ftvalid/test"
)
tdl = DataLoader(tds,
batch_size=batsize,
shuffle=True,
collate_fn=partial(autocollate, pad_value=0))
advdl = DataLoader(advds,
batch_size=batsize,
shuffle=True,
collate_fn=partial(autocollate, pad_value=0))
ftdl = DataLoader(ftds,
batch_size=batsize,
shuffle=True,
collate_fn=partial(autocollate, pad_value=0))
vdl = DataLoader(vds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(autocollate, pad_value=0))
fvdl = DataLoader(fvds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(autocollate, pad_value=0))
xdl = DataLoader(xds,
batch_size=batsize,
shuffle=False,
collate_fn=partial(autocollate, pad_value=0))
tt.tock("data loaded")
tt.tick("creating model")
trainm, advtrainm, testm = create_model(
encoder_name=encoder,
fl_vocab=flenc.vocab,
abs_fl_vocab=absflenc.vocab,
dec_layers=numlayers,
dec_dim=hdim,
dec_heads=numheads,
dropout=dropout,
smoothing=smoothing,
maxlen=maxlen,
numbeam=numbeam,
abs_id=absflenc.vocab["@ABS@"],
entropycontrib=entropycontrib,
)
tt.tock("model created")
# run a batch of data through the model
if localtest:
batch = next(iter(tdl))
out = trainm(*batch)
print(out)
out = testm(*batch)
print(out)
# region pretrain on all domains
metrics = make_array_of_metrics("loss", "ce", "elem_acc", "tree_acc")
advmetrics = make_array_of_metrics("adv_loss", "adv_elem_acc",
"adv_tree_acc")
vmetrics = make_array_of_metrics("seq_acc", "tree_acc")
xmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params
if k not in exclude_params]
tt.msg("different param groups")
encparams = [
v for k, v in trainable_params if k.startswith("model.model.encoder")
]
otherparams = [
v for k, v in trainable_params
if not k.startswith("model.model.encoder")
]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{
"params": encparams,
"lr": lr * enclrmul
}, {
"params": otherparams
}]
optim = torch.optim.Adam(paramgroups, lr=lr, weight_decay=wreg)
advoptim = torch.optim.Adam(advtrainm.parameters(),
lr=advlr,
weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
trainm.parameters(), gradnorm)
advclipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
advtrainm.parameters(), gradnorm)
eyt = q.EarlyStopper(vmetrics[1],
patience=patience,
min_epochs=10,
more_is_better=True,
remember_f=lambda: deepcopy(trainm.model))
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(
steps=t_max - warmup) >> 0.
advlr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(
steps=t_max - warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
advlr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(optim, lr_schedule)
advlr_schedule = q.sched.LRSchedule(advoptim, advlr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
advtrainbatch = partial(q.train_batch,
on_before_optim_step=[advclipgradnorm])
trainepoch = partial(
adv_train_epoch,
model=trainm,
dataloader=tdl,
optim=optim,
losses=metrics,
advmodel=advtrainm,
advdataloader=advdl,
advoptim=advoptim,
advlosses=advmetrics,
_train_batch=trainbatch,
_adv_train_batch=advtrainbatch,
device=device,
on_end=[lambda: lr_schedule.step(), lambda: advlr_schedule.step()],
advsteps=advsteps)
validepoch = partial(q.test_epoch,
model=testm,
dataloader=vdl,
losses=vmetrics,
device=device,
on_end=[lambda: eyt.on_epoch_end()])
if not nopretrain:
tt.tick("pretraining")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=pretrainepochs,
check_stop=[lambda: eyt.check_stop()])
tt.tock("done pretraining")
if eyt.get_remembered() is not None:
tt.msg("reloaded")
trainm.model = eyt.get_remembered()
testm.model = eyt.get_remembered()
# endregion
# region finetune
ftmetrics = make_array_of_metrics("loss", "ce", "elem_acc", "tree_acc")
ftvmetrics = make_array_of_metrics("seq_acc", "tree_acc")
ftxmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params
if k not in exclude_params]
tt.msg("different param groups")
encparams = [
v for k, v in trainable_params if k.startswith("model.model.encoder")
]
otherparams = [
v for k, v in trainable_params
if not k.startswith("model.model.encoder")
]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{
"params": encparams,
"lr": ftlr * enclrmul
}, {
"params": otherparams
}]
ftoptim = torch.optim.Adam(paramgroups, lr=ftlr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(
trainm.parameters(), gradnorm)
eyt = q.EarlyStopper(ftvmetrics[1],
patience=patience,
min_epochs=10,
more_is_better=True,
remember_f=lambda: deepcopy(trainm.model))
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(
steps=t_max - warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(ftoptim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch,
model=trainm,
dataloader=ftdl,
optim=ftoptim,
losses=ftmetrics,
_train_batch=trainbatch,
device=device,
on_end=[lambda: lr_schedule.step()])
validepoch = partial(q.test_epoch,
model=testm,
dataloader=fvdl,
losses=ftvmetrics,
device=device,
on_end=[lambda: eyt.on_epoch_end()])
tt.tick("training")
q.run_training(run_train_epoch=trainepoch,
run_valid_epoch=validepoch,
max_epochs=epochs,
check_stop=[lambda: eyt.check_stop()])
tt.tock("done training")
if eyt.get_remembered() is not None:
tt.msg("reloaded")
trainm.model = eyt.get_remembered()
testm.model = eyt.get_remembered()
# endregion
tt.tick("testing")
validresults = q.test_epoch(model=testm,
dataloader=fvdl,
losses=ftvmetrics,
device=device)
testresults = q.test_epoch(model=testm,
dataloader=xdl,
losses=ftxmetrics,
device=device)
print(validresults)
print(testresults)
tt.tock("tested")
if printtest:
predm = testm.model
predm.to(device)
c, t = 0, 0
for testbatch in iter(xdl):
input_ids = testbatch[0]
output_ids = testbatch[1]
input_ids = input_ids.to(device)
ret = predm.generate(
input_ids,
attention_mask=input_ids != predm.config.pad_token_id,
max_length=maxlen)
inp_strs = [
nltok.decode(input_idse,
skip_special_tokens=True,
clean_up_tokenization_spaces=False)
for input_idse in input_ids
]
out_strs = [
flenc.vocab.tostr(rete.to(torch.device("cpu"))) for rete in ret
]
gold_strs = [
flenc.vocab.tostr(output_idse.to(torch.device("cpu")))
for output_idse in output_ids
]
for x, y, g in zip(inp_strs, out_strs, gold_strs):
print(" ")
print(f"'{x}'\n--> {y}\n <=> {g}")
if y == g:
c += 1
else:
print("NOT SAME")
t += 1
print(f"seq acc: {c/t}")
# testout = q.eval_loop(model=testm, dataloader=xdl, device=device)
# print(testout)
print("done")
# settings.update({"train_seqacc": losses[]})
for metricarray, datasplit in zip([ftmetrics, ftvmetrics, ftxmetrics],
["train", "valid", "test"]):
for metric in metricarray:
settings[f"{datasplit}_{metric.name}"] = metric.get_epoch_error()
# print(settings)
return settings
def run(domain="restaurants",
lr=0.001,
ptlr=0.0001,
enclrmul=0.1,
cosinelr=False,
ptcosinelr=False,
warmup=0.,
ptwarmup=0.,
batsize=20,
ptbatsize=50,
epochs=100,
ptepochs=100,
dropout=0.1,
wreg=1e-9,
gradnorm=3,
smoothing=0.,
patience=5,
gpu=-1,
seed=123456789,
dataseed=12345678,
datatemp=0.33,
ptN=3000,
tokenmaskp=0.,
spanmaskp=0.,
spanmasklamda=2.2,
treemaskp=0.,
encoder="bart-large",
numlayers=6,
hdim=600,
numheads=8,
maxlen=50,
localtest=False,
printtest=False,
):
settings = locals().copy()
print(locals())
random.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
tt = q.ticktock("script")
device = torch.device("cpu") if gpu < 0 else torch.device(gpu)
tt.tick("loading data")
tds, vds, xds, nltok, flenc = load_ds(domain=domain, nl_mode=encoder)
tdl = DataLoader(tds, batch_size=batsize, shuffle=True, collate_fn=partial(autocollate, pad_value=1))
vdl = DataLoader(vds, batch_size=batsize, shuffle=False, collate_fn=partial(autocollate, pad_value=1))
xdl = DataLoader(xds, batch_size=batsize, shuffle=False, collate_fn=partial(autocollate, pad_value=1))
tt.tock("data loaded")
tt.tick("creating grammar dataset generator")
pcfg = build_grammar(tds, vds)
ptds = PCFGDataset(pcfg, N=ptN, seed=seed, temperature=datatemp, maxlen=100)
tt.tock("created dataset generator")
tt.tick("creating model")
trainm, testm, pretrainm = create_model(encoder_name=encoder,
dec_vocabsize=flenc.vocab.number_of_ids(),
dec_layers=numlayers,
dec_dim=hdim,
dec_heads=numheads,
dropout=dropout,
smoothing=smoothing,
maxlen=maxlen,
tensor2tree=partial(_tensor2tree, D=flenc.vocab)
)
tt.tock("model created")
# run a batch of data through the model
if localtest:
print("generated dataset")
print(ptds[0])
print(ptds[0])
allexamples = []
for i in tqdm(range(len(ptds))):
allexamples.append(ptds[i])
uniqueexamples = set([str(x) for x in allexamples])
print(f"{100*len(uniqueexamples)/len(allexamples)}% unique examples ({len(uniqueexamples)}/{len(allexamples)})")
ptds.advance_seed()
print(ptds[0])
allexamples = list(ptds.examples)
uniqueexamples2 = set([str(x) for x in allexamples])
print(f"{100*len(uniqueexamples2)/len(allexamples)}% unique examples ({len(uniqueexamples2)}/{len(allexamples)})")
print(f"{len(uniqueexamples & uniqueexamples2)}/{len(uniqueexamples | uniqueexamples2)} overlap")
print("---")
batch = next(iter(tdl))
out = trainm(*batch)
print(out)
out = testm(*batch)
print(out)
# region pretraining
# setup data perturbation
tokenmasker = TokenMasker(p=tokenmaskp, seed=dataseed) if tokenmaskp > 0 else lambda x: x
spanmasker = SpanMasker(p=spanmaskp, lamda=spanmasklamda, seed=dataseed) if spanmaskp > 0 else lambda x: x
treemasker = SubtreeMasker(p=treemaskp, seed=dataseed) if treemaskp > 0 else lambda x: x
perturbed_ptds = ptds\
.map(lambda x: (treemasker(x), x))\
.map(lambda x: (flenc.convert(x[0], "tokens"),
flenc.convert(x[1], "tokens")))\
.map(lambda x: (spanmasker(tokenmasker(x[0])), x[1]))
perturbed_ptds_tokens = perturbed_ptds
perturbed_ptds = perturbed_ptds\
.map(lambda x: (flenc.convert(x[0], "tensor"),
flenc.convert(x[1], "tensor")))
if localtest:
allex = []
allperturbedex = []
_nepo = 50
print(f"checking {_nepo}, each {ptN} generated examples")
for _e in tqdm(range(_nepo)):
for i in range(len(perturbed_ptds_tokens)):
ex = str(ptds[i])
perturbed_ex = perturbed_ptds_tokens[i]
perturbed_ex = f"{' '.join(perturbed_ex[0])}->{' '.join(perturbed_ex[1])}"
allex.append(ex)
allperturbedex.append(perturbed_ex)
ptds.advance_seed()
uniqueex = set(allex)
uniqueperturbedex = set(allperturbedex)
print(f"{len(uniqueex)}/{len(allex)} unique examples")
print(f"{len(uniqueperturbedex)}/{len(allperturbedex)} unique perturbed examples")
ptdl = DataLoader(perturbed_ptds, batch_size=ptbatsize, shuffle=True, collate_fn=partial(autocollate, pad_value=1))
ptmetrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
ptparams = pretrainm.parameters()
ptoptim = torch.optim.Adam(ptparams, lr=ptlr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(trainm.parameters(), gradnorm)
t_max = ptepochs
print(f"Total number of pretraining updates: {t_max} .")
if ptcosinelr:
lr_schedule = q.sched.Linear(steps=ptwarmup) >> q.sched.Cosine(steps=t_max-ptwarmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=ptwarmup) >> 1.
lr_schedule = q.sched.LRSchedule(ptoptim, lr_schedule)
pttrainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
pttrainepoch = partial(q.train_epoch, model=pretrainm, dataloader=ptdl, optim=ptoptim, losses=ptmetrics,
_train_batch=pttrainbatch, device=device, on_end=[lambda: lr_schedule.step(),
lambda: ptds.advance_seed()])
tt.tick("pretraining")
q.run_training(run_train_epoch=pttrainepoch, max_epochs=ptepochs)
tt.tock("done pretraining")
# endregion
# region finetuning
metrics = make_array_of_metrics("loss", "elem_acc", "seq_acc", "tree_acc")
vmetrics = make_array_of_metrics("seq_acc", "tree_acc")
xmetrics = make_array_of_metrics("seq_acc", "tree_acc")
trainable_params = list(trainm.named_parameters())
exclude_params = set()
# exclude_params.add("model.model.inp_emb.emb.weight") # don't train input embeddings if doing glove
if len(exclude_params) > 0:
trainable_params = [(k, v) for k, v in trainable_params if k not in exclude_params]
tt.msg("different param groups")
encparams = [v for k, v in trainable_params if k.startswith("model.model.encoder")]
otherparams = [v for k, v in trainable_params if not k.startswith("model.model.encoder")]
if len(encparams) == 0:
raise Exception("No encoder parameters found!")
paramgroups = [{"params": encparams, "lr": lr * enclrmul},
{"params": otherparams}]
optim = torch.optim.Adam(paramgroups, lr=lr, weight_decay=wreg)
clipgradnorm = lambda: torch.nn.utils.clip_grad_norm_(trainm.parameters(), gradnorm)
eyt = q.EarlyStopper(vmetrics[1], patience=patience, min_epochs=10, more_is_better=True, remember_f=lambda: deepcopy(trainm.model))
t_max = epochs
print(f"Total number of updates: {t_max} .")
if cosinelr:
lr_schedule = q.sched.Linear(steps=warmup) >> q.sched.Cosine(steps=t_max-warmup) >> 0.
else:
lr_schedule = q.sched.Linear(steps=warmup) >> 1.
lr_schedule = q.sched.LRSchedule(optim, lr_schedule)
trainbatch = partial(q.train_batch, on_before_optim_step=[clipgradnorm])
trainepoch = partial(q.train_epoch, model=trainm, dataloader=tdl, optim=optim, losses=metrics,
_train_batch=trainbatch, device=device, on_end=[lambda: lr_schedule.step(), lambda: eyt.on_epoch_end()])
validepoch = partial(q.test_epoch, model=testm, dataloader=vdl, losses=vmetrics, device=device)
tt.tick("training")
q.run_training(run_train_epoch=trainepoch, run_valid_epoch=validepoch, max_epochs=epochs, check_stop=[lambda: eyt.check_stop()])
tt.tock("done training")
if eyt.get_remembered() is not None:
trainm.model = eyt.get_remembered()
testm.model = eyt.get_remembered()
tt.tick("testing")
testresults = q.test_epoch(model=testm, dataloader=xdl, losses=xmetrics, device=device)
print(testresults)
tt.tock("tested")
if printtest:
predm = testm.model
predm.to(device)
c, t = 0, 0
for testbatch in iter(xdl):
input_ids = testbatch[0]
output_ids = testbatch[1]
input_ids = input_ids.to(device)
ret = predm.generate(input_ids, attention_mask=input_ids != predm.config.pad_token_id,
max_length=maxlen)
inp_strs = [nltok.decode(input_idse, skip_special_tokens=True, clean_up_tokenization_spaces=False) for input_idse in input_ids]
out_strs = [flenc.vocab.tostr(rete.to(torch.device("cpu"))) for rete in ret]
gold_strs = [flenc.vocab.tostr(output_idse.to(torch.device("cpu"))) for output_idse in output_ids]
for x, y, g in zip(inp_strs, out_strs, gold_strs):
print(" ")
print(f"'{x}'\n--> {y}\n <=> {g}")
if y == g:
c += 1
else:
print("NOT SAME")
t += 1
print(f"seq acc: {c/t}")
# testout = q.eval_loop(model=testm, dataloader=xdl, device=device)
# print(testout)
print("done")
# settings.update({"train_seqacc": losses[]})
for metricarray, datasplit in zip([metrics, vmetrics, xmetrics], ["train", "valid", "test"]):
for metric in metricarray:
settings[f"{datasplit}_{metric.name}"] = metric.get_epoch_error()
# print(settings)
return settings
def run(lr=20.,
dropout=0.2,
dropconnect=0.2,
gradnorm=0.25,
epochs=25,
embdim=200,
encdim=200,
numlayers=2,
tieweights=False,
seqlen=35,
batsize=20,
eval_batsize=80,
cuda=False,
gpu=0,
test=False):
tt = q.ticktock("script")
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
tt.tick("loading data")
train_batches, valid_batches, test_batches, D = \
load_data(batsize=batsize, eval_batsize=eval_batsize,
seqlen=VariableSeqlen(minimum=5, maximum_offset=10, mu=seqlen, sigma=0))
tt.tock("data loaded")
print("{} batches in train".format(len(train_batches)))
tt.tick("creating model")
dims = [embdim] + ([encdim] * numlayers)
m = RNNLayer_LM(*dims, worddic=D, dropout=dropout,
tieweights=tieweights).to(device)
if test:
for i, batch in enumerate(train_batches):
y = m(batch[0])
if i > 5:
break
print(y.size())
loss = q.LossWrapper(q.CELoss(mode="logits"))
validloss = q.LossWrapper(q.CELoss(mode="logits"))
validlosses = [validloss, PPLfromCE(validloss)]
testloss = q.LossWrapper(q.CELoss(mode="logits"))
testlosses = [testloss, PPLfromCE(testloss)]
for l in [loss] + validlosses + testlosses: # put losses on right device
l.loss.to(device)
optim = torch.optim.SGD(m.parameters(), lr=lr)
train_batch_f = partial(
q.train_batch,
on_before_optim_step=[
lambda: torch.nn.utils.clip_grad_norm_(m.parameters(), gradnorm)
])
lrp = torch.optim.lr_scheduler.ReduceLROnPlateau(optim,
mode="min",
factor=1 / 4,
patience=0,
verbose=True)
lrp_f = lambda: lrp.step(validloss.get_epoch_error())
train_epoch_f = partial(q.train_epoch,
model=m,
dataloader=train_batches,
optim=optim,
losses=[loss],
device=device,
_train_batch=train_batch_f)
valid_epoch_f = partial(q.test_epoch,
model=m,
dataloader=valid_batches,
losses=validlosses,
device=device,
on_end=[lrp_f])
tt.tock("created model")
tt.tick("training")
q.run_training(train_epoch_f,
valid_epoch_f,
max_epochs=epochs,
validinter=1)
tt.tock("trained")
tt.tick("testing")
testresults = q.test_epoch(model=m,
dataloader=test_batches,
losses=testlosses,
device=device)
print(testresults)
tt.tock("tested")
