def __init__(self, config):
super().__init__(config)
self.autoencoder = AutoEncoder(config.autoencoder)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_autoencoder = config.autoencoder.n_epochs
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
def __init__(self, config):
super().__init__(config)
self.mod_0_to_1 = MLP(config.mod_0_to_1)
self.mod_1_to_0 = MLP(config.mod_1_to_0)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_cross_modality = config.cross_modality.n_epochs
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
class JointEncodingOption1(ExperimentOption1):
def __init__(self, config):
super().__init__(config)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
def train_jointencoder_batch(self, inp_0, inp_1):
return self.jointencoder.train(inp_0, inp_1)
def train_readout_batch(self, inp_0, inp_1, target):
encoding = self.jointencoder(inp_0, inp_1, what=['encoding'])['encoding']
return self.readout.train(encoding, target)
def get_readout_loss(self, inp_0, inp_1, target):
encoding = self.jointencoder(inp_0, inp_1, what=['encoding'])['encoding']
return self.readout(encoding, target, what=['loss'])['loss']
def get_image_reconstructions(self, inp_0, inp_1):
return self.jointencoder(
inp_0,
inp_1,
what=['reconstructions']
)['reconstruction_0']
class CrossModalityOption2(ExperimentOption2):
def __init__(self, config):
super().__init__(config)
self.autoencoder = AutoEncoder(config.autoencoder)
self.mod_0_to_1 = MLP(config.mod_0_to_1)
self.mod_1_to_0 = MLP(config.mod_1_to_0)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_autoencoder = config.autoencoder.n_epochs
self.n_epoch_cross_modality = config.cross_modality.n_epochs
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
def train_autoencoder_batch(self, inp):
return self.autoencoder.train(inp)
def train_cross_modality_batch(self, inp_0, inp_1):
encoding = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
loss_0 = self.mod_0_to_1.train(encoding, inp_1)
loss_1 = self.mod_1_to_0.train(inp_1, encoding)
return loss_0, loss_1
def train_jointencoder_batch(self, inp_0, inp_1):
encoding = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
prediction_0 = self.mod_0_to_1(encoding, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
return self.jointencoder.train(prediction_1, prediction_0)
def train_readout_batch(self, inp_0, inp_1, target):
encoding_a = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
prediction_0 = self.mod_0_to_1(encoding_a, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
encoding_b = self.jointencoder(prediction_1, prediction_0, what=['encoding'])['encoding']
return self.readout.train(encoding_b, target)
def get_readout_loss(self, inp_0, inp_1, target):
encoding_a = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
prediction_0 = self.mod_0_to_1(encoding_a, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
encoding_b = self.jointencoder(prediction_1, prediction_0, what=['encoding'])['encoding']
return self.readout(encoding_b, target, what=['loss'])['loss']
def get_image_reconstructions(self, inp_0, inp_1):
encoding = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
jointencoder_reconstruction = self.jointencoder(
self.mod_1_to_0(inp_1, what=['output'])['output'],
self.mod_0_to_1(encoding, what=['output'])['output'],
what=['reconstructions']
)['reconstruction_0'] * self.encoding_std + self.encoding_mean
return self.autoencoder.get_reconstruction(jointencoder_reconstruction)
class CrossModalityOption1(ExperimentOption1):
def __init__(self, config):
super().__init__(config)
self.mod_0_to_1 = MLP(config.mod_0_to_1)
self.mod_1_to_0 = MLP(config.mod_1_to_0)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_cross_modality = config.cross_modality.n_epochs
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
def train_cross_modality_batch(self, inp_0, inp_1):
loss_0 = self.mod_0_to_1.train(inp_0, inp_1)
loss_1 = self.mod_1_to_0.train(inp_1, inp_0)
return loss_0, loss_1
def train_jointencoder_batch(self, inp_0, inp_1):
prediction_0 = self.mod_0_to_1(inp_0, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
return self.jointencoder.train(prediction_1, prediction_0)
def train_readout_batch(self, inp_0, inp_1, target):
prediction_0 = self.mod_0_to_1(inp_0, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
encoding = self.jointencoder(prediction_1, prediction_0, what=['encoding'])['encoding']
return self.readout.train(encoding, target)
def get_readout_loss(self, inp_0, inp_1, target):
prediction_0 = self.mod_0_to_1(inp_0, what=['output'])['output']
prediction_1 = self.mod_1_to_0(inp_1, what=['output'])['output']
encoding = self.jointencoder(prediction_1, prediction_0, what=['encoding'])['encoding']
return self.readout(encoding, target, what=['loss'])['loss']
def get_image_reconstructions(self, inp_0, inp_1):
return self.jointencoder(
self.mod_1_to_0(inp_1, what=['output'])['output'],
self.mod_0_to_1(inp_0, what=['output'])['output'],
what=['reconstructions']
)['reconstruction_0']
class JointEncodingOption2(ExperimentOption2):
def __init__(self, config):
super().__init__(config)
self.autoencoder = AutoEncoder(config.autoencoder)
self.jointencoder = JointEncoder(config.jointencoder)
self.readout = MLP(config.readout)
self.n_epoch_autoencoder = config.autoencoder.n_epochs
self.n_epoch_jointencoder = config.jointencoder.n_epochs
self.n_epoch_readout = config.readout.n_epochs
def train_autoencoder_batch(self, inp):
return self.autoencoder.train(inp)
def train_jointencoder_batch(self, inp_0, inp_1):
encoding = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
return self.jointencoder.train(encoding, inp_1)
def train_readout_batch(self, inp_0, inp_1, target):
encoding_a = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
encoding_b = self.jointencoder(encoding_a, inp_1, what=['encoding'])['encoding']
return self.readout.train(encoding_b, target)
def get_readout_loss(self, inp_0, inp_1, target):
encoding_a = (self.autoencoder(inp_0, what=['encoding'])['encoding'] - self.encoding_mean) / self.encoding_std
encoding_b = self.jointencoder(encoding_a, inp_1, what=['encoding'])['encoding']
return self.readout(encoding_b, target, what=['loss'])['loss']
def get_image_reconstructions(self, inp_0, inp_1):
return self.autoencoder.get_reconstruction(
self.jointencoder(
(self.autoencoder(
inp_0,
what=['encoding']
)['encoding'] - self.encoding_mean) / self.encoding_std,
inp_1,
what=['reconstructions']
)['reconstruction_0'] * self.encoding_std + self.encoding_mean
)