def run_experiment(
lr=-1.,
enclrmul=-1.,
smoothing=-1.,
gradnorm=2,
batsize=-1,
epochs=-1, # probably 11 is enough
warmup=3,
cosinelr=False,
dataset="default",
maxsize=-1,
seed=-1,
hdim=-1,
numlayers=-1,
numheads=-1,
dropout=-1.,
sidedrop=0.,
bertname="vanilla",
testcode=False,
userelpos=False,
trainonvalidonly=False,
evaltrain=False,
gpu=-1,
recomputedata=False,
mode="normal", # "normal", "vib"
priorweight=-1.,
):
settings = locals().copy()
settings["patience"] = -1
settings["validinter"] = 1
ranges = {
"dataset": [
"scan/random", "scan/length", "scan/add_jump",
"scan/add_turn_left", "scan/mcd1", "scan/mcd2", "scan/mcd3"
],
"dropout": [0.1, 0.25, 0.5],
"seed": [42, 87646464, 456852],
"epochs": [15],
"batsize": [60],
"hdim": [768],
"numheads": [12],
"numlayers": [6],
"lr": [0.0001],
"enclrmul": [0.1], # use 1.
"smoothing": [0., 0.1],
# "patience": [-1],
# "warmup": [20],
# "validinter": [2],
# "gradacc": [1],
# "sidedrop": [0., 0.5, 0.9],
}
if bertname.startswith("none") or bertname == "vanilla":
ranges["lr"] = [0.0001]
ranges["enclrmul"] = [1.]
ranges["epochs"] = [40]
ranges["hdim"] = [384]
ranges["numheads"] = [6]
ranges["batsize"] = [512]
# ranges["validinter"] = [3]
ranges["dropout"] = [0.1]
ranges["smoothing"] = [0.]
if dataset.startswith("cfq"):
settings["maxsize"] = 200
elif dataset.startswith("scan"):
settings["maxsize"] = 50
if mode in ("vib", "vibatt"):
ranges["priorweight"] = [0.0001, 0.001, 0.01]
for k in ranges:
if k in settings:
if isinstance(settings[k], str) and settings[k] != "default":
ranges[k] = [settings[k]]
elif isinstance(settings[k], (int, float)) and settings[k] >= 0:
ranges[k] = [settings[k]]
else:
pass
# raise Exception(f"something wrong with setting '{k}'")
del settings[k]
def checkconfig(spec):
return True
print(__file__)
p = __file__ + f".baseline.{dataset}"
q.run_experiments_random(run,
ranges,
path_prefix=p,
check_config=checkconfig,
**settings)
def run_experiment(
lr=-1.,
enclrmul=-1.,
smoothing=-1.,
gradnorm=2,
batsize=-1,
epochs=-1, # probably 11 is enough
patience=100,
validinter=-1,
warmup=3,
cosinelr=False,
dataset="default",
datasets="both",
mode="normal",
maxsize=-1,
seed=-1,
hdim=-1,
numlayers=-1,
numheads=-1,
dropout=-1.,
worddropout=-1.,
bertname="vanilla",
testcode=False,
userelpos=False,
trainonvalidonly=False,
evaltrain=False,
gpu=-1,
recomputedata=False,
mcdropout=-1,
):
settings = locals().copy()
del settings["datasets"]
ranges = {
"dataset": [
"scan/random", "scan/length", "scan/add_jump",
"scan/add_turn_left", "scan/mcd1", "scan/mcd2", "scan/mcd3",
"cfq/mcd1", "cfq/mcd2", "cfq/mcd3"
],
# "dataset": ["scan/random", "scan/length", "scan/add_jump", "scan/add_turn_left", "scan/mcd1", "scan/mcd2", "scan/mcd3"],
# "dataset": ["cfq/mcd1", "cfq/mcd2", "cfq/mcd3"],
# "dataset": ["scan/length", "scan/add_jump", "scan/add_turn_left", "scan/mcd3"],
"dropout": [0.1, 0.25, 0.5],
"worddropout": [0.],
"seed": [42, 87646464, 456852],
"epochs": [40, 25],
# "epochs": [25],
"batsize": [256, 128],
# "batsize": [100],
"hdim": [384],
"numheads": [12],
"numlayers": [2],
# "numlayers": [2],
"lr": [0.0005],
"enclrmul": [1.], # use 1.
"smoothing": [0.],
"validinter": [1],
"mcdropout": [0, 5],
}
if datasets == "both":
pass
elif datasets == "cfq":
ranges["dataset"] = ["cfq/mcd1", "cfq/mcd2", "cfq/mcd3"]
elif datasets == "scan":
ranges["dataset"] = [
"scan/random", "scan/length", "scan/add_jump",
"scan/add_turn_left", "scan/mcd1", "scan/mcd2", "scan/mcd3"
]
elif datasets == "mcd":
ranges["dataset"] = [
"cfq/mcd1", "cfq/mcd2", "cfq/mcd3", "scan/mcd1", "scan/mcd2",
"scan/mcd3"
]
elif datasets == "nonmcd":
ranges["dataset"] = [
"scan/random", "scan/length", "scan/add_jump", "scan/add_turn_left"
]
for k in ranges:
if k in settings:
if isinstance(settings[k], str) and settings[k] != "default":
ranges[k] = [settings[k]]
elif isinstance(settings[k], (int, float)) and settings[k] >= 0:
ranges[k] = [settings[k]]
else:
pass
# raise Exception(f"something wrong with setting '{k}'")
del settings[k]
def checkconfig(spec):
if spec["dataset"].startswith("cfq"):
if spec["epochs"] != 25 or spec["batsize"] != 128:
return False
elif spec["dataset"].startswith("scan"):
if spec["epochs"] != 40 or spec["batsize"] != 256:
return False
return True
print(__file__)
p = __file__ + f".baseline.{dataset}"
q.run_experiments_random(run,
ranges,
path_prefix=None,
check_config=checkconfig,
**settings)
def run_experiment(
domain="default", #
mode="baseline", # "baseline", "ltr", "uniform", "binary"
probthreshold=-1.,
lr=-1.,
enclrmul=-1.,
batsize=-1,
epochs=-1,
hdim=-1,
numlayers=-1,
numheads=-1,
dropout=-1.,
noreorder=False,
trainonvalid=False,
seed=-1,
gpu=-1,
patience=-1,
gradacc=-1,
cosinelr=False,
warmup=-1,
gradnorm=3.,
validinter=-1,
maxsteps=90,
maxsize=90,
numbered=False,
testcode=False,
):
settings = locals().copy()
ranges = {
"domain": [
"socialnetwork", "blocks", "calendar", "housing", "restaurants",
"publications", "recipes", "basketball"
],
"dropout": [0.0, 0.1, 0.2, 0.3, 0.4],
"epochs": [121],
"batsize": [50],
"hdim": [366, 768],
"numheads": [6, 12],
"numlayers": [6, 8, 12],
"lr": [0.0001, 0.000025],
"enclrmul": [1., 0.1], # use 1.
"seed": [87646464],
"patience": [-1],
"warmup": [20],
"validinter": [15],
"gradacc": [1],
}
if mode == "baseline": # baseline
ranges["validinter"] = [5]
else:
# ranges["domain"] = ["blocks", "calendar", "housing", "restaurants", "publications", "recipes", "basketball"]
# ranges["domain"] = ["calendar", "publications", "recipes"]
ranges["batsize"] = [30]
ranges["dropout"] = [0.0, 0.1, 0.2] # use 0.
# ranges["lr"] = [0.0001] # use 0.000025
ranges["validinter"] = [20]
ranges["epochs"] = [301]
ranges["hdim"] = [768]
ranges["numlayers"] = [6]
ranges["numheads"] = [12]
ranges["probthreshold"] = [0.]
ranges["lr"] = [0.000025]
if mode == "ltr":
ranges["lr"] = [0.0001, 0.000025]
ranges["warmup"] = [50]
ranges["epochs"] = [501]
ranges["validinter"] = [25]
ranges["gradacc"] = [10]
ranges["hdim"] = [768]
ranges["numlayers"] = [6]
ranges["numheads"] = [12]
for k in ranges:
if k in settings:
if isinstance(settings[k], str) and settings[k] != "default":
ranges[k] = [settings[k]]
elif isinstance(settings[k], (int, float)) and settings[k] >= 0:
ranges[k] = [settings[k]]
else:
pass
# raise Exception(f"something wrong with setting '{k}'")
del settings[k]
def checkconfig(spec):
if spec["hdim"] == 366 and spec["numheads"] != 6:
return False
if spec["hdim"] == 768 and spec["numheads"] != 12:
return False
return True
print(__file__)
p = __file__ + f".baseline.{domain}"
q.run_experiments_random(run,
ranges,
path_prefix=p,
check_config=checkconfig,
**settings)