for (opt, dr, gbs, gsf) in options:
#for opt in ['rmsprop']#, 'adadelta', nest(1,0.9)]:
# for dr in [0.001]#drop_rates[0:-1]:
# for gbs in [0.001]#gauss_base_sigmas[0:3]:
# for gsf in [1.5]#[1, 1.1, 2]:
for (l1, l2) in secondary:
counter += 1
print(str(counter)+" of "+str(len(options)*len(secondary)))
name = "o-"+str(opt)+"-dr-"+str(dr)+"-gbs-"+str(gbs)+"-gsf-"+str(gsf)+"-l1-"+str(l1)+"-l2-"+str(l2)
#if not ae.get_from_catalog("layer_sizes", "ed_"+name):
ae.enc_opt = opt
ae.drop_rate = dr
ae.sigma_base = gbs
ae.sigma_fact = gsf
ae.l1 = l1
ae.l2 = l2
ae.new_encdecs(use_dropout=(dr != 0), use_noise=(gbs != 0))
update_status(ae, "current", "training encdec "+name)
h = ae.pretrain(name=name)
info = {"acc_pre_"+str(i): float(e[-1][1]) for (i,e) in enumerate(h)}
info.update({("loss_pre_"+str(i)): float(e[-1][0]) for (i,e) in enumerate(h)})
ae.update_catalog("ed_"+name,info)
update_status(ae, "history", {"name": name,
"loss": losscomb(0, h),
"acc": losscomb(1, h),
"date": time_str()})
if continue_encdecs:
opt, dr, gbs, gsf, l2 = 'adadelta', 0.001, 0.001, 2, 0.001
name = "o-"+str(opt)+"-dr-"+str(dr)+"-gbs-"+str(gbs)+"-gsf-"+str(gsf)+"-l1-"+str(0)+"-l2-"+str(l2)
ae.enc_opt = opt
ae.drop_rate = dr
n = nest(1,0.9)
counter = 0
options = [('adadelta', 0.001, 0.001, 2)]
ls = [0.001]
secondary = [(0, l2) for l2 in ls]
for (opt, dr, gbs, gsf) in options:
for (l1, l2) in secondary:
counter += 1
print(str(counter)+" of "+str(len(options)*len(secondary)))
name = "o-"+str(opt)+"-dr-"+str(dr)+"-gbs-"+str(gbs)+"-gsf-"+str(gsf)+"-l1-"+str(l1)+"-l2-"+str(l2)
ae.enc_opt = opt
ae.drop_rate = dr
ae.sigma_base = gbs
ae.sigma_fact = gsf
ae.l1 = l1
ae.l2 = l2
ae.new_encdecs(use_dropout=(dr != 0), use_noise=(gbs != 0))
print("training encdec "+name)
h = ae.pretrain(name=name)
info = {"acc_pre_"+str(i): float(e[-1][1]) for (i,e) in enumerate(h)}
info.update({("loss_pre_"+str(i)): float(e[-1][0]) for (i,e) in enumerate(h)})
ae.update_catalog("ed_"+name,info)
### FINETUNE ###
cur_left_out = nsm.cross_val_keys[nsm.cross_val_index]
ae.load_encdecs("best")
ae.finetune(name="x-"+cur_left_out+"-val", train_encdecs=True, test_data=nsm.data[cur_left_out])
info = {}
for key in ae.previous_data:
print(key)
arr = np.array(ae.previous_data[key])
phl = len(phase_names)
ls = [0.001]
secondary = [(0, l2) for l2 in ls]
for (opt, dr, gbs, gsf) in options:
for (l1, l2) in secondary:
counter += 1
print(
str(counter) + " of " + str(len(options) * len(secondary)))
name = "o-" + str(opt) + "-dr-" + str(dr) + "-gbs-" + str(
gbs) + "-gsf-" + str(gsf) + "-l1-" + str(
l1) + "-l2-" + str(l2)
ae.enc_opt = opt
ae.drop_rate = dr
ae.sigma_base = gbs
ae.sigma_fact = gsf
ae.l1 = l1
ae.l2 = l2
ae.new_encdecs(use_dropout=(dr != 0), use_noise=(gbs != 0))
print("training encdec " + name)
h = ae.pretrain(name=name)
info = {
"acc_pre_" + str(i): float(e[-1][1])
for (i, e) in enumerate(h)
}
info.update({("loss_pre_" + str(i)): float(e[-1][0])
for (i, e) in enumerate(h)})
ae.update_catalog("ed_" + name, info)
### FINETUNE ###
cur_left_out = nsm.cross_val_keys[nsm.cross_val_index]
ae.load_encdecs("best")
ae.finetune(name="x-" + cur_left_out + "-val",
train_encdecs=True,
