def train_with_noise(vari, rep, ignoreme): # pylint: disable=unused-argument
"""Entry point"""
train_pwl = MNISTData.load_train().to_pwl().restrict_to(set(
range(10))).rescale()
test_pwl = MNISTData.load_test().to_pwl().restrict_to(set(
range(10))).rescale()
layers_and_nonlins = (
(90, 'tanh'),
(90, 'tanh'),
(90, 'tanh'),
(90, 'tanh'),
(90, 'tanh'),
)
layers = [lyr[0] for lyr in layers_and_nonlins]
nonlins = [lyr[1] for lyr in layers_and_nonlins]
nonlins.append('tanh') # output
#layer_names = [f'{lyr[1]} (layer {idx})' for idx, lyr in enumerate(layers_and_nonlins)]
layer_names = [
f'Layer {idx+1}' for idx, lyr in enumerate(layers_and_nonlins)
]
layer_names.insert(0, 'Input')
layer_names.append('Output')
network = FeedforwardLarge.create(input_dim=train_pwl.input_dim,
output_dim=train_pwl.output_dim,
weights=wi.GaussianWeightInitializer(
mean=0, vari=0.3, normalize_dim=0),
biases=wi.ZerosWeightInitializer(),
layer_sizes=layers,
nonlinearity=nonlins
#layer_sizes=[500, 200]
)
_lr = 0.1
trainer = tnr.GenericTrainer(
train_pwl=train_pwl,
test_pwl=test_pwl,
teacher=FFTeacher(),
batch_size=30,
learning_rate=_lr,
optimizer=torch.optim.SGD(
[p for p in network.parameters() if p.requires_grad], lr=_lr
), #torch.optim.Adam([p for p in network.parameters() if p.requires_grad], lr=0.003),
criterion=mycrits.meansqerr #torch.nn.CrossEntropyLoss()#
)
#pca3d_throughtrain.FRAMES_PER_TRAIN = 4
#pca3d_throughtrain.SKIP_TRAINS = 0
#pca3d_throughtrain.NUM_FRAME_WORKERS = 6
dig = npmp.NPDigestor(f'TRMCN_{rep}_{vari}', 8)
savedir = os.path.join(SAVEDIR, f'variance_{vari}', f'repeat_{rep}')
dtt_training_dir = os.path.join(savedir, 'dtt')
pca_training_dir = os.path.join(savedir, 'pca')
pca3d_training_dir = os.path.join(savedir, 'pca3d')
pr_training_dir = os.path.join(savedir, 'pr')
svm_training_dir = os.path.join(savedir, 'svm')
satur_training_dir = os.path.join(savedir, 'saturation')
trained_net_dir = os.path.join(savedir, 'trained_model')
pca_throughtrain_dir = os.path.join(savedir, 'pca_throughtrain')
logpath = os.path.join(savedir, 'log.txt')
(trainer.reg(tnr.EpochsTracker()).reg(tnr.EpochsStopper(0.2)).reg(
tnr.DecayTracker()).reg(tnr.DecayStopper(5)).reg(
tnr.LRMultiplicativeDecayer())
#.reg(tnr.DecayOnPlateau())
#.reg(tnr.DecayEvery(5))
.reg(tnr.AccuracyTracker(1, 1000, True)).reg(
tnr.WeightNoiser(
wi.GaussianWeightInitializer(mean=0, vari=vari),
(lambda ctx: ctx.model.layers[-1].weight.data.detach()), 'scale',
(lambda noise: wi.GaussianWeightInitializer(0, noise.vari * 0.5)
)))
#.reg(tnr.OnEpochCaller.create_every(dtt.during_training_ff(dtt_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(pca_3d.during_training(pca3d_training_dir, True, dig, plot_kwargs={'layer_names': layer_names}), start=500, skip=100))
#.reg(tnr.OnEpochCaller.create_every(pca_ff.during_training(pca_training_dir, True, dig), skip=100))
.reg(
tnr.OnEpochCaller.create_every(pr.during_training_ff(
pr_training_dir, True, dig),
skip=1))
#.reg(tnr.OnEpochCaller.create_every(svm.during_training_ff(svm_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(satur.during_training(satur_training_dir, True, dig), skip=100))
.reg(
tnr.OnEpochCaller.create_every(tnr.save_model(trained_net_dir),
skip=100))
#.reg(pca3d_throughtrain.PCAThroughTrain(pca_throughtrain_dir, layer_names, True))
.reg(tnr.OnFinishCaller(lambda *args, **kwargs: dig.join())).reg(
tnr.CopyLogOnFinish(logpath)).reg(
tnr.ZipDirOnFinish(dtt_training_dir)).reg(
tnr.ZipDirOnFinish(pca_training_dir)).reg(
tnr.ZipDirOnFinish(pca3d_training_dir)).reg(
tnr.ZipDirOnFinish(pr_training_dir)).reg(
tnr.ZipDirOnFinish(svm_training_dir)).reg(
tnr.ZipDirOnFinish(satur_training_dir)).reg(
tnr.ZipDirOnFinish(trained_net_dir)))
trainer.train(network)
dig.archive_raw_inputs(os.path.join(savedir, 'digestor_raw.zip'))
def main():
"""Entry point"""
nets = cu.FluentShape(28 * 28).verbose()
network = FeedforwardComplex(INPUT_DIM, OUTPUT_DIM, [
nets.linear_(HIDDEN_DIM),
nets.tanh(),
nets.linear_(OUTPUT_DIM),
nets.tanh()
])
train_pwl = MNISTData.load_train().to_pwl().restrict_to(set(
range(10))).rescale()
test_pwl = MNISTData.load_test().to_pwl().restrict_to(set(
range(10))).rescale()
layer_names = ('Input', 'Hidden', 'Output')
trainer = tnr.GenericTrainer(
train_pwl=train_pwl,
test_pwl=test_pwl,
teacher=FFTeacher(),
batch_size=45,
learning_rate=0.001,
optimizer=torch.optim.Adam(
[p for p in network.parameters() if p.requires_grad], lr=0.001),
criterion=mycrits.meansqerr #torch.nn.CrossEntropyLoss()
)
dig = npmp.NPDigestor('train_one_complex', 35)
dtt_training_dir = os.path.join(SAVEDIR, 'dtt')
pca_training_dir = os.path.join(SAVEDIR, 'pca')
pca3d_training_dir = os.path.join(SAVEDIR, 'pca3d')
pr_training_dir = os.path.join(SAVEDIR, 'pr')
svm_training_dir = os.path.join(SAVEDIR, 'svm')
satur_training_dir = os.path.join(SAVEDIR, 'saturation')
trained_net_dir = os.path.join(SAVEDIR, 'trained_model')
pca_throughtrain_dir = os.path.join(SAVEDIR, 'pca_throughtrain')
wds_training_dir = os.path.join(SAVEDIR, 'weightdeltas')
logpath = os.path.join(SAVEDIR, 'log.txt')
(trainer.reg(tnr.EpochsTracker()).reg(tnr.EpochsStopper(3)).reg(
tnr.DecayTracker()).reg(tnr.DecayStopper(8)).reg(
tnr.LRMultiplicativeDecayer()).reg(tnr.DecayOnPlateau()).
reg(tnr.AccuracyTracker(5, 1000, True)).reg(
tnr.WeightNoiser(
wi.GaussianWeightInitializer(mean=0,
vari=0.1),
(lambda ctx: ctx.model.layers[0].action.weight.data.detach()),
'scale',
(lambda noise: wi.GaussianWeightInitializer(0, noise.vari * 0.5)
))).reg(
tnr.OnEpochCaller.create_every(dtt.during_training_ff(
dtt_training_dir, True, dig),
skip=100))
#.reg(tnr.OnEpochCaller.create_every(pca_3d.during_training(pca3d_training_dir, True, dig, plot_kwargs={'layer_names': layer_names}), start=1000, skip=1000))
.reg(
tnr.OnEpochCaller.create_every(
pca_ff.during_training(pca_training_dir, True, dig),
skip=100)).reg(
tnr.OnEpochCaller.create_every(
pr.during_training_ff(pr_training_dir, True, dig),
skip=100)).reg(
tnr.OnEpochCaller.create_every(
svm.during_training_ff(svm_training_dir, True,
dig),
skip=100)).reg(
tnr.OnEpochCaller.create_every(
satur.during_training(
satur_training_dir, True, dig),
skip=100)).reg(
tnr.OnEpochCaller.create_every(
tnr.save_model(trained_net_dir),
skip=100)).
reg(
wds.Binned2Norm(
(lambda ctx: ctx.model.layers[0].action.weight.data.detach()),
dig, wds_training_dir, 'Induced Changes in $W^{(1)}$'))
#.reg(pca3d_throughtrain.PCAThroughTrain(pca_throughtrain_dir, layer_names, True, layer_indices=plot_layers))
.reg(tnr.OnFinishCaller(lambda *args, **kwargs: dig.join())).reg(
tnr.CopyLogOnFinish(logpath)).reg(
tnr.ZipDirOnFinish(dtt_training_dir)).reg(
tnr.ZipDirOnFinish(pca_training_dir)).reg(
tnr.ZipDirOnFinish(pca3d_training_dir)).reg(
tnr.ZipDirOnFinish(pr_training_dir)).reg(
tnr.ZipDirOnFinish(svm_training_dir)).reg(
tnr.ZipDirOnFinish(satur_training_dir)).reg(
tnr.ZipDirOnFinish(trained_net_dir)))
trainer.train(network)
dig.archive_raw_inputs(os.path.join(SAVEDIR, 'digestor_raw.zip'))
def train_with_noise(vari, rep, pr_repeats, ignoreme): # pylint: disable=unused-argument
"""Entry point"""
train_pwl = GaussianSpheresPWLP.create(epoch_size=30000,
input_dim=INPUT_DIM,
output_dim=2,
cube_half_side_len=2,
num_clusters=10,
std_dev=0.2,
mean=0,
min_sep=0.4,
force_split=True)
test_pwl = train_pwl
nets = cu.FluentShape(INPUT_DIM).verbose()
mywi = wi.WICombine([
wi.RectangularEyeWeightInitializer(1),
wi.GaussianWeightInitializer(mean=0, vari=0.3)
])
network = FeedforwardComplex(INPUT_DIM, train_pwl.output_dim, [
nets.linear_(DIM, weights_init=mywi),
nets.nonlin('leakyrelu'),
nets.linear_(DIM, weights_init=mywi),
nets.nonlin('leakyrelu'),
nets.linear_(DIM, weights_init=mywi),
nets.nonlin('leakyrelu'),
nets.linear_(DIM, weights_init=mywi),
nets.nonlin('leakyrelu'),
nets.linear_(train_pwl.output_dim),
nets.nonlin('leakyrelu'),
])
_lr = 0.01
trainer = tnr.GenericTrainer(
train_pwl=train_pwl,
test_pwl=test_pwl,
teacher=FFTeacher(),
batch_size=20,
learning_rate=_lr,
optimizer=torch.optim.SGD(
[p for p in network.parameters() if p.requires_grad], lr=_lr),
criterion=mycrits.hubererr #torch.nn.CrossEntropyLoss()#
)
#pca3d_throughtrain.FRAMES_PER_TRAIN = 4
#pca3d_throughtrain.SKIP_TRAINS = 0
#pca3d_throughtrain.NUM_FRAME_WORKERS = 6
dig = npmp.NPDigestor(f'TRMCN_{rep}_{vari}', 4)
savedir = os.path.join(SAVEDIR, f'variance_{vari}', f'repeat_{rep}')
shared.filetools.deldir(savedir)
dtt_training_dir = os.path.join(savedir, 'dtt')
pca_training_dir = os.path.join(savedir, 'pca')
pca3d_training_dir = os.path.join(savedir, 'pca3d')
pr_training_dir = os.path.join(savedir, 'pr')
svm_training_dir = os.path.join(savedir, 'svm')
satur_training_dir = os.path.join(savedir, 'saturation')
trained_net_dir = os.path.join(savedir, 'trained_model')
pca_throughtrain_dir = os.path.join(savedir, 'pca_throughtrain')
acts_training_dir = os.path.join(savedir, 'acts')
logpath = os.path.join(savedir, 'log.txt')
(trainer.reg(tnr.EpochsTracker()).reg(tnr.EpochsStopper(300)).reg(
tnr.EpochProgress(5, hint_end_epoch=10)).reg(tnr.DecayTracker()).reg(
tnr.DecayStopper(10)).reg(tnr.InfOrNANDetecter()).reg(
tnr.InfOrNANStopper()).reg(
tnr.LRMultiplicativeDecayer(factor=0.9))
#.reg(tnr.DecayOnPlateau(verbose=False))
.reg(tnr.DecayEvery(1, verbose=False)).reg(
tnr.AccuracyTracker(1,
1000,
True,
savepath=os.path.join(savedir, 'accuracy.json'))))
if ALL_LAYERS_NOISED:
tonoise = list(range(1, len(network.layers)))
else:
tonoise = [len(network.layers) - 2]
noisestyle = 'add'
def layer_fetcher(lyr):
return lambda ctx: ctx.model.layers[lyr].action.weight.data.detach()
noisedecayer = lambda noise: wi.GaussianWeightInitializer(
0, noise.vari * 0.9)
for lyr in tonoise:
if network.layers[lyr].is_module:
trainer.reg(
tnr.WeightNoiser(
wi.GaussianWeightInitializer(mean=0, vari=vari),
layer_fetcher(lyr), noisestyle, noisedecayer))
if rep < pr_repeats:
trainer.reg(
tnr.OnEpochCaller.create_every(pr.during_training_ff(
pr_training_dir, True, dig),
skip=100))
(trainer
#.reg(tnr.OnEpochCaller.create_every(dtt.during_training_ff(dtt_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(pca_3d.during_training(pca3d_training_dir, True, dig, plot_kwargs={'layer_names': layer_names}), start=500, skip=100))
#.reg(tnr.OnEpochCaller.create_every(pca_ff.during_training(pca_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(pr.during_training_ff(pr_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(svm.during_training_ff(svm_training_dir, True, dig), skip=100))
#.reg(tnr.OnEpochCaller.create_every(satur.during_training(satur_training_dir, True, dig), skip=100))
.reg(tnr.OnEpochCaller.create_every(measacts.during_training(acts_training_dir, dig, meta={'time': time.time(), 'noised_layers': tonoise, 'variance': vari, 'repeat': rep}), skip=100))
.reg(tnr.OnEpochCaller.create_every(tnr.save_model(trained_net_dir), skip=100))
#.reg(pca3d_throughtrain.PCAThroughTrain(pca_throughtrain_dir, layer_names, True))
.reg(tnr.OnFinishCaller(lambda *args, **kwargs: dig.join()))
.reg(tnr.CopyLogOnFinish(logpath))
.reg(tnr.ZipDirOnFinish(dtt_training_dir))
.reg(tnr.ZipDirOnFinish(pca_training_dir))
.reg(tnr.ZipDirOnFinish(pca3d_training_dir))
.reg(tnr.ZipDirOnFinish(pr_training_dir))
.reg(tnr.ZipDirOnFinish(svm_training_dir))
.reg(tnr.ZipDirOnFinish(satur_training_dir))
.reg(tnr.ZipDirOnFinish(trained_net_dir))
)
result = trainer.train(network)
dig.archive_raw_inputs(os.path.join(savedir, 'digestor_raw.zip'))
if result['inf_or_nan']:
print('[TMCN] Inf or NAN detected - repeating run')
shared.filetools.deldir(savedir)
def main():
"""Entry point"""
cu.DEFAULT_LINEAR_BIAS_INIT = wi.ZerosWeightInitializer()
cu.DEFAULT_LINEAR_WEIGHT_INIT = wi.GaussianWeightInitializer(
mean=0, vari=0.3, normalize_dim=0)
nets = cu.FluentShape(32 * 32 * 3).verbose()
network = FeedforwardComplex(INPUT_DIM, OUTPUT_DIM, [
nets.linear_(32 * 32 * 6),
nets.nonlin('isrlu'),
nets.linear_(500),
nets.nonlin('tanh'),
nets.linear_(250),
nets.nonlin('tanh'),
nets.linear_(250),
nets.nonlin('tanh'),
nets.linear_(100),
nets.tanh(),
nets.linear_(100),
nets.tanh(),
nets.linear_(100),
nets.tanh(),
nets.linear_(OUTPUT_DIM),
nets.nonlin('isrlu'),
])
train_pwl = CIFARData.load_train().to_pwl().restrict_to(set(
range(10))).rescale()
test_pwl = CIFARData.load_test().to_pwl().restrict_to(set(
range(10))).rescale()
layer_names = ('input', 'FC -> 32*32*6 (ISRLU)', 'FC -> 500 (tanh)',
'FC -> 250 (tang)', 'FC -> 250 (tanh)', 'FC -> 100 (tanh)',
'FC -> 100 (tanh)', 'FC -> 100 (tanh)',
f'FC -> {OUTPUT_DIM} (ISRLU)')
plot_layers = tuple(i for i in range(2, len(layer_names) - 1))
trainer = tnr.GenericTrainer(
train_pwl=train_pwl,
test_pwl=test_pwl,
teacher=FFTeacher(),
batch_size=45,
learning_rate=0.001,
optimizer=torch.optim.Adam(
[p for p in network.parameters() if p.requires_grad], lr=0.001),
criterion=torch.nn.CrossEntropyLoss())
pca3d_throughtrain.FRAMES_PER_TRAIN = 1
pca3d_throughtrain.SKIP_TRAINS = 16
pca3d_throughtrain.NUM_FRAME_WORKERS = 1
dig = npmp.NPDigestor('train_one_complex', 5)
dtt_training_dir = os.path.join(SAVEDIR, 'dtt')
pca_training_dir = os.path.join(SAVEDIR, 'pca')
pca3d_training_dir = os.path.join(SAVEDIR, 'pca3d')
pr_training_dir = os.path.join(SAVEDIR, 'pr')
svm_training_dir = os.path.join(SAVEDIR, 'svm')
satur_training_dir = os.path.join(SAVEDIR, 'saturation')
trained_net_dir = os.path.join(SAVEDIR, 'trained_model')
pca_throughtrain_dir = os.path.join(SAVEDIR, 'pca_throughtrain')
logpath = os.path.join(SAVEDIR, 'log.txt')
(trainer.reg(tnr.EpochsTracker()).reg(tnr.EpochsStopper(STOP_EPOCH)).reg(
tnr.DecayTracker()).reg(tnr.DecayStopper(8)).reg(
tnr.EpochProgress(print_every=120, hint_end_epoch=STOP_EPOCH)).reg(
tnr.LRMultiplicativeDecayer()).reg(
tnr.DecayOnPlateau(patience=3)).reg(
tnr.AccuracyTracker(1, 1000, True)).reg(
tnr.OnEpochCaller.create_every(
dtt.during_training_ff(dtt_training_dir, True,
dig),
skip=5)).reg(
tnr.OnEpochCaller.create_every(
pca_3d.during_training(
pca3d_training_dir,
True,
dig,
plot_kwargs={
'layer_names': layer_names
}),
start=10,
skip=100)).
reg(
tnr.OnEpochCaller.create_every(
pca_ff.during_training(pca_training_dir, True, dig), skip=5)).reg(
tnr.OnEpochCaller.create_every(
pr.during_training_ff(pr_training_dir,
True,
dig,
labels=False),
skip=5)).reg(
tnr.OnEpochCaller.create_every(
svm.during_training_ff(svm_training_dir, True,
dig),
skip=5)).reg(
tnr.OnEpochCaller.create_every(
satur.during_training(
satur_training_dir, True, dig),
skip=5)).reg(
tnr.OnEpochCaller.create_every(
tnr.save_model(trained_net_dir),
skip=5))
#.reg(pca3d_throughtrain.PCAThroughTrain(pca_throughtrain_dir, layer_names, True, layer_indices=plot_layers))
.reg(tnr.OnFinishCaller(lambda *args, **kwargs: dig.join())).reg(
tnr.ZipDirOnFinish(dtt_training_dir)).reg(
tnr.ZipDirOnFinish(pca_training_dir)).reg(
tnr.ZipDirOnFinish(pca3d_training_dir)).reg(
tnr.ZipDirOnFinish(pr_training_dir)).reg(
tnr.ZipDirOnFinish(svm_training_dir)).reg(
tnr.ZipDirOnFinish(satur_training_dir)).reg(
tnr.ZipDirOnFinish(trained_net_dir)).reg(
tnr.CopyLogOnFinish(logpath)))
trainer.train(network)
dig.archive_raw_inputs(os.path.join(SAVEDIR, 'digestor_raw.zip'))