def test_TrainerEmbedding_cached(self):
set_seed(3)
criterion = TripletLossSampler(nn.TripletMarginLoss())
trainer = TrainerEmbedding(
self.model,
criterion=criterion,
data_loader=self.data_loader,
optimizer=self.optimizer,
scheduler=self.scheduler,
accuracy_measure=AccuracyEmbedding(cache=True))
trainer.open_monitor(offline=True)
# TripletLoss is not deterministic; fix the seed
set_seed(4)
loss_cached = trainer.full_forward_pass(train=True)
accuracy_cached = trainer.update_accuracy(train=True)
trainer.accuracy_measure.cache = False
trainer.accuracy_measure.reset()
set_seed(4)
loss = trainer.full_forward_pass(train=True)
accuracy = trainer.update_accuracy(train=True)
self.assertAlmostEqual(loss_cached.item(), loss.item())
self.assertAlmostEqual(accuracy_cached, accuracy)
def train_matching_pursuit(dataset_cls=MNIST, n_epochs=5):
# Typically, the 'out_features', the second parameter of MatchingPursuit
# model, should be greater than the 'in_features'.
# In case of MNIST, it works even with the smaller values, but the
# resulting embedding vector is dense.
model = MatchingPursuit(784, 256, solver=BasisPursuitADMM())
# model = LISTA(784, 128)
data_loader = DataLoader(dataset_cls,
transform=TransformDefault.mnist(
normalize=None
))
criterion = LossPenalty(nn.MSELoss(), lambd=model.lambd)
optimizer, scheduler = get_optimizer_scheduler(model)
trainer = TrainMatchingPursuit(model,
criterion=criterion,
data_loader=data_loader,
optimizer=optimizer,
scheduler=scheduler,
accuracy_measure=AccuracyEmbedding())
trainer.restore()
n_epochs = max(n_epochs - trainer.epoch, 0)
trainer.train(n_epochs=n_epochs, mutual_info_layers=0)
return trainer.model
def test_TrainerAutoencoder_cached(self):
set_seed(3)
model = AutoencoderLinear(784, 64)
trainer = TrainerAutoencoder(
model,
criterion=nn.BCEWithLogitsLoss(),
data_loader=self.data_loader,
optimizer=self.optimizer,
scheduler=self.scheduler,
accuracy_measure=AccuracyEmbedding(cache=True))
trainer.open_monitor(offline=True)
# TripletLoss is not deterministic; fix the seed
set_seed(4)
loss_cached = trainer.full_forward_pass(train=True)
accuracy_cached = trainer.update_accuracy(train=True)
trainer.accuracy_measure.cache = False
trainer.accuracy_measure.reset()
set_seed(4)
loss = trainer.full_forward_pass(train=True)
accuracy = trainer.update_accuracy(train=True)
self.assertAlmostEqual(loss_cached.item(), loss.item())
self.assertAlmostEqual(accuracy_cached, accuracy)
def test_matching_pursuit(dataset_cls=MNIST):
# vanilla Matching Pursuit: the weights are fixed, no training
model = MatchingPursuit(784, 2048)
data_loader = DataLoader(dataset_cls,
eval_size=10000,
transform=TransformDefault.mnist(
normalize=None
))
trainer = TestMatchingPursuit(model,
criterion=nn.MSELoss(),
data_loader=data_loader,
optimizer=OptimizerStub(),
accuracy_measure=AccuracyEmbedding())
trainer.train(n_epochs=1, mutual_info_layers=0)
def train_autoencoder(n_epoch=60, dataset_cls=MNIST):
model = AutoencoderLinear(784, 128)
data_loader = DataLoader(dataset_cls,
transform=TransformDefault.mnist(normalize=None))
criterion = nn.BCEWithLogitsLoss()
optimizer, scheduler = get_optimizer_scheduler(model)
trainer = TrainerAutoencoder(
model,
criterion=criterion,
data_loader=data_loader,
optimizer=optimizer,
scheduler=scheduler,
accuracy_measure=AccuracyEmbedding(cache=True))
# trainer.restore() # uncomment to restore the saved state
trainer.monitor.advanced_monitoring(level=MonitorLevel.SIGNAL_TO_NOISE)
trainer.train(n_epochs=n_epoch, mutual_info_layers=0)
def __init__(self,
model: nn.Module,
criterion: nn.Module,
data_loader: DataLoader,
optimizer: Optimizer,
scheduler: Union[_LRScheduler, ReduceLROnPlateau] = None,
accuracy_measure: Accuracy = AccuracyEmbedding(),
**kwargs):
if not isinstance(accuracy_measure, AccuracyEmbedding):
raise ValueError("'accuracy_measure' must be of instance "
f"{AccuracyEmbedding.__name__}")
super().__init__(model=model,
criterion=criterion,
data_loader=data_loader,
optimizer=optimizer,
scheduler=scheduler,
accuracy_measure=accuracy_measure,
**kwargs)
def train_lista(dataset_cls=MNIST):
model_ref = train_matching_pursuit(dataset_cls=dataset_cls)
model = LISTA(784, out_features=model_ref.out_features)
data_loader = DataLoader(dataset_cls,
transform=TransformDefault.mnist(
normalize=None
))
criterion = nn.MSELoss()
optimizer, scheduler = get_optimizer_scheduler(model)
trainer = TrainLISTA(model,
model_reference=model_ref,
criterion=criterion,
data_loader=data_loader,
optimizer=optimizer,
scheduler=scheduler,
accuracy_measure=AccuracyEmbedding())
# trainer.monitor.advanced_monitoring(level=MonitorLevel.FULL)
trainer.train(n_epochs=10, mutual_info_layers=0)
def __init__(self,
model: nn.Module,
criterion: nn.Module,
data_loader: DataLoader,
accuracy_measure: Accuracy = None,
mutual_info=None,
env_suffix='',
checkpoint_dir=CHECKPOINTS_DIR):
if torch.cuda.is_available():
model.cuda()
self.model = model
self.criterion = criterion
self.data_loader = data_loader
self.train_loader = data_loader.get(train=True)
if mutual_info is None:
mutual_info = MutualInfoNeuralEstimation(data_loader)
self.mutual_info = mutual_info
self.checkpoint_dir = Path(checkpoint_dir)
self.timer = timer
self.timer.init(batches_in_epoch=len(self.train_loader))
self.timer.set_epoch(0)
criterion_name = self.criterion.__class__.__name__
if isinstance(criterion, LossPenalty):
criterion_name = f"{criterion_name}(" \
f"{criterion.criterion.__class__.__name__})"
self.env_name = f"{time.strftime('%Y.%m.%d')} " \
f"{model.__class__.__name__}: " \
f"{data_loader.dataset_cls.__name__} " \
f"{self.__class__.__name__} " \
f"{criterion_name}"
env_suffix = env_suffix.lstrip(' ')
if env_suffix:
self.env_name = f'{self.env_name} {env_suffix}'
if accuracy_measure is None:
if isinstance(self.criterion, PairLossSampler):
accuracy_measure = AccuracyEmbedding()
else:
# cross entropy loss
accuracy_measure = AccuracyArgmax()
self.accuracy_measure = accuracy_measure
self.monitor = self._init_monitor(mutual_info)
self.online = self._init_online_measures()
self.best_score = 0.