def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_007 import Encoder, Decoder self.encoder = Encoder(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads()
def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.md_loss = MeanDistanceLoss() self.er_loss = EntropyRegularizationLoss() self.d_loss = DistanceLoss() # Model from recon.cifar10.cnn_model_001 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads()
class Experiment004(Experiment003): """Enc-MLP-Dec - MeanDistanceLoss - ResNet """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.md_loss = MeanDistanceLoss() self.er_loss = EntropyRegularizationLoss() self.d_loss = DistanceLoss() # Model from recon.cifar10.cnn_model_001 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() def train(self, x_l, y, x_u): self._train(x_l, (x_l, y), y) self._train(x_u, (x_l, y)) def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss loss += self.md_loss(self.encoder.hiddens[-1]) # MD loss self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_mlp.update() def cleargrads(self, ): self.encoder.cleargrads() self.mlp.cleargrads()
class Experiment004(Experiment003): """Enc-MLP-Dec - MeanDistanceLoss - ResNet """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.md_loss = MeanDistanceLoss() self.er_loss = EntropyRegularizationLoss() self.d_loss = DistanceLoss() # Model from recon.cifar10.cnn_model_001 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() def train(self, x_l, y, x_u): self._train(x_l, (x_l, y), y) self._train(x_u, (x_l, y)) def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss loss += self.md_loss(self.encoder.hiddens[-1]) # MD loss self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_mlp.update() def cleargrads(self, ): self.encoder.cleargrads() self.mlp.cleargrads()
class Experiment002(Experiment000): """Enc-MLP-Dec - Encoder contains linear function - ResNet - U-Net """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_002 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss x_rec = self.decoder(h, self.encoder.hiddens) loss += self.recon_loss(x, x_rec) self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() self.optimizer_mlp.update()
class Experiment002(Experiment000): """Enc-MLP-Dec - Encoder contains linear function - ResNet - U-Net """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_002 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss x_rec = self.decoder(h, self.encoder.hiddens) loss += self.recon_loss(x, x_rec) self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() self.optimizer_mlp.update()
class Experiment009(Experiment008): """Enc-MLP-Dec - ConvPool-CNN-C (Springenberg et al., 2014, Salimans&Kingma (2016)) - Encoder contains classifier """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_007 import Encoder, Decoder self.encoder = Encoder(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder y_pred = self.encoder(x) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss x_rec = self.decoder(y_pred) loss += self.recon_loss(x, x_rec) self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() def cleargrads(self, ): self.encoder.cleargrads() self.decoder.cleargrads()
class Experiment009(Experiment008): """Enc-MLP-Dec - ConvPool-CNN-C (Springenberg et al., 2014, Salimans&Kingma (2016)) - Encoder contains classifier """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_007 import Encoder, Decoder self.encoder = Encoder(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder y_pred = self.encoder(x) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss x_rec = self.decoder(y_pred) loss += self.recon_loss(x, x_rec) self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() def cleargrads(self, ): self.encoder.cleargrads() self.decoder.cleargrads()
def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_007 import Encoder, Decoder self.encoder = Encoder(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads()
def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.md_loss = MeanDistanceLoss() self.er_loss = EntropyRegularizationLoss() self.d_loss = DistanceLoss() # Model from recon.cifar10.cnn_model_001 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads()
class Experiment006(Experiment005): """Enc-MLP-Dec - Encoder contains linear function - BN of classification is different from BN of encoding - Mean-only BN """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_004 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads()
class Experiment006(Experiment005): """Enc-MLP-Dec - Encoder contains linear function - BN of classification is different from BN of encoding - Mean-only BN """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_004 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads()
class Experiment007(Experiment000): """Enc-MLP-Dec - Encoder contains linear function - ResNet - BN of classification is different from BN of encoding """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_005 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def train(self, x_l, y, x_u): self._train(x_l, (x_l, y), y) self._train(x_u, (x_l, y)) def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x, enc=False) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss h = self.encoder(x, enc=True) x_rec = self.decoder(h) loss += self.recon_loss(x, x_rec) \ + reduce(lambda u, v: u + v, [self.recon_loss(u, v) \ for u, v in zip(self.encoder.hiddens, self.decoder.hiddens[::-1])]) # RC loss self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() self.optimizer_mlp.update() def test(self, x, y): h = self.encoder(x, enc=False, test=True) y_pred = self.mlp(h) acc = F.accuracy(y_pred, y) return acc
class Experiment007(Experiment000): """Enc-MLP-Dec - Encoder contains linear function - ResNet - BN of classification is different from BN of encoding """ def __init__(self, device=None, learning_rate=1e-3, act=F.relu, n_cls=10): # Settings self.device = device self.act = act self.learning_rate = learning_rate self.n_cls = n_cls # Losses self.recon_loss = ReconstructionLoss() self.er_loss = EntropyRegularizationLoss() # Model from recon.cifar10.cnn_model_005 import Encoder, MLP, Decoder self.encoder = Encoder(device, act) self.mlp = MLP(device, act) self.decoder = Decoder(device, act) self.encoder.to_gpu(device) if self.device else None self.mlp.to_gpu(device) if self.device else None self.decoder.to_gpu(device) if self.device else None # Optimizer self.optimizer_enc = optimizers.Adam(learning_rate) self.optimizer_enc.setup(self.encoder) self.optimizer_enc.use_cleargrads() self.optimizer_mlp = optimizers.Adam(learning_rate) self.optimizer_mlp.setup(self.mlp) self.optimizer_mlp.use_cleargrads() self.optimizer_dec = optimizers.Adam(learning_rate) self.optimizer_dec.setup(self.decoder) self.optimizer_dec.use_cleargrads() def train(self, x_l, y, x_u): self._train(x_l, (x_l, y), y) self._train(x_u, (x_l, y)) def _train(self, x, xy, y_0=None): x_, y_ = xy # Encoder/Decoder h = self.encoder(x, enc=False) y_pred = self.mlp(h) loss = 0 loss += self.er_loss(y_pred) # ER loss if y_0 is not None: loss += F.softmax_cross_entropy(y_pred, y_0) # CE loss h = self.encoder(x, enc=True) x_rec = self.decoder(h) loss += self.recon_loss(x, x_rec) \ + reduce(lambda u, v: u + v, [self.recon_loss(u, v) \ for u, v in zip(self.encoder.hiddens, self.decoder.hiddens[::-1])]) # RC loss self.cleargrads() loss.backward() self.optimizer_enc.update() self.optimizer_dec.update() self.optimizer_mlp.update() def test(self, x, y): h = self.encoder(x, enc=False, test=True) y_pred = self.mlp(h) acc = F.accuracy(y_pred, y) return acc