def test_fuse_modalities_2():
"""
With 3 experts and a batch size of 3, the mixture selection should select each of the experts one time.
"""
class_dim = 3
batch_size = 3
num_mods = 3
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname,
method='planar_mixture',
attributes={
'num_mods': num_mods,
'class_dim': class_dim,
'device': 'cpu',
'batch_size': batch_size,
'weighted_mixture': False
},
dataset=DATASET)
model: PlanarMixtureMMVae = mst.mm_vae
enc_mods: Mapping[str, EncModPlanarMixture] = {
mod_str: EncModPlanarMixture(None,
None,
z0=None,
zk=torch.ones(
(batch_size, class_dim)) * numbr)
for numbr, mod_str in zip(range(num_mods), mst.modalities)
}
joint_zk = model.mixture_component_selection(enc_mods,
'm0_m1_m2',
weight_joint=False)
assert torch.all(
joint_zk == Tensor([[0., 0., 0.], [1., 1., 1.], [2., 2., 2.]]))
def test_run_epochs_polymnist(method: str):
"""
Test if the main training loop runs.
Assert if the total_test loss is constant. If the assertion fails, it means that the model or the evaluation has
changed, perhaps involuntarily.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
# todo implement calc likelihood for flow based methods
calc_nll = False if method in [
'mofop', 'planar_mixture', 'pfom', 'pope', 'fomfop', 'fomop',
'poe', 'gfm', 'planar_vae', 'sylvester_vae_noflow', 'iwmogfm',
'iwmogfm2', 'iwmogfm3', 'iwmogfm_amortized', 'iwmogfm_old'
] else True
# calc_nll = False
mst = set_me_up(
tmpdirname,
dataset='polymnist',
method=method,
attributes={
'calc_nll': calc_nll,
"K": 5,
"dir_clf": Path("/tmp/trained_clfs_polyMNIST")
# 'num_mods': 1
# 'num_flows': 1
})
trainer = PolymnistTrainer(mst)
test_results = trainer.run_epochs()
def test_fuse_modalities_4():
"""
With 3 experts and a batch size of 3, the mixture selection should select each of the experts one time.
"""
class_dim = 3
batch_size = 3
num_mods = 3
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname,
method='moe',
attributes={
'num_mods': num_mods,
'class_dim': class_dim,
'device': 'cpu',
'batch_size': batch_size,
'weighted_mixture': False
},
dataset=DATASET)
model: MOEMMVae = mst.mm_vae
mus = torch.ones((num_mods, batch_size, class_dim))
mus[0] = mus[0] * 0
mus[2] = mus[2] * 2
logvars = torch.zeros((num_mods, batch_size, class_dim))
w_modalities = torch.ones((num_mods, )) * (1 / 3)
joint_distr = mixture_component_selection(mst.flags, mus, logvars,
w_modalities)
assert torch.all(joint_distr.mu == Tensor([[0., 0., 0.], [1., 1., 1.],
[2., 2., 2.]]))
def test_run_planar_mixture_no_flow():
"""
Test if the main training loop runs.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
method = 'planar_mixture'
additional_attrs = {'num_flows': 0, 'num_mods': 1}
mst = set_me_up(tmpdirname, method, attributes=additional_attrs)
trainer = PolymnistTrainer(mst)
test_results = trainer.run_epochs()
def test_run_epochs_mnistsvhntext(method: str):
"""
Test if the main training loop runs.
Assert if the total_test loss is constant. If the assertion fails, it means that the model or the evaluation has
changed, perhaps involuntarily.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
# todo implement calc likelihood for flow based methods
calc_nll = method not in ['planar_mixture', 'pfom', 'pope']
mst = set_me_up(tmpdirname,
dataset='mnistsvhntext',
method=method,
attributes={
'calc_nll': True,
"dir_clf": Path("/tmp/trained_clfs_mst")
})
trainer = mnistsvhnTrainer(mst)
test_results = trainer.run_epochs()
def test_fuse_modalities_3():
"""
test that during the mixture selection, the batches don't get mixed up.
If 3 experts have a zk of:
Tensor([[0., 0., 0.],
[1., 1., 1.],
[2., 2., 2.]])
(of shape (batch_size, class dim)).
Then the mixture selection should select the first batch from expert 1, the second batch from expert 2 and the
third batch from expert 3, giving the same result.
"""
class_dim = 3
batch_size = 3
num_mods = 3
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname,
method='planar_mixture',
attributes={
'num_mods': num_mods,
'class_dim': class_dim,
'device': 'cpu',
'batch_size': batch_size,
'weighted_mixture': False
},
dataset=DATASET)
model: PlanarMixtureMMVae = mst.mm_vae
enc_mod_zk = torch.ones((batch_size, class_dim))
enc_mod_zk[0] = enc_mod_zk[0] * 0
enc_mod_zk[2] = enc_mod_zk[2] * 2
enc_mods: Mapping[str, EncModPlanarMixture] = {
mod_str: EncModPlanarMixture(None, None, z0=None, zk=enc_mod_zk)
for mod_str in mst.modalities
}
joint_zk = model.mixture_component_selection(enc_mods,
'm0_m1_m2',
weight_joint=False)
assert torch.all(
joint_zk == Tensor([[0., 0., 0.], [1., 1., 1.], [2., 2., 2.]]))
def test_run_epochs_celeba(method: str):
"""
Test if the main training loop runs.
Assert if the total_test loss is constant. If the assertion fails, it means that the model or the evaluation has
changed, perhaps involuntarily.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
# todo implement calc likelihood for flow based methods
calc_nll = method not in ['planar_mixture', 'pfom', 'pope']
mst = set_me_up(
tmpdirname,
dataset='celeba',
method=method,
attributes={
'calc_nll': True,
'use_clf': True,
'batch_size': 2,
# 'calc_prd': True
})
trainer = CelebaTrainer(mst)
test_results = trainer.run_epochs()
def test_static_results_2mods(method: str):
"""
Test if the results are constant. If the assertion fails, it means that the model or the evaluation has
changed, perhaps involuntarily.
"""
static_results = json2dict(
Path('static_results.json'))['static_results_2mod']
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname,
method=method,
attributes={
'num_flows': 0,
'num_mods': 2,
'deterministic': True,
'device': 'cpu',
'steps_per_training_epoch': 1,
'factorized_representation': False
})
trainer = PolymnistTrainer(mst)
test_results = trainer.run_epochs()
assert np.round(test_results.joint_div,
1) == np.round(static_results[method]['joint_div'], 1)
assert np.round(test_results.klds['m0'],
1) == np.round(static_results[method]['klds'], 1)
assert np.round(test_results.lhoods['m0']['m0'],
1) == np.round(static_results[method]['lhoods'], 1)
assert np.round(test_results.log_probs['m0'],
0) == np.round(static_results[method]['log_probs'], 0)
assert np.round(test_results.total_loss,
0) == np.round(static_results[method]['total_loss'], 0)
assert np.round(test_results.lr_eval['m0']['accuracy'],
2) == np.round(static_results[method]['lr_eval'], 2)
assert np.round(test_results.latents['m0']['latents_class']['mu'],
2) == np.round(
static_results[method]['latents_class']['mu'], 2)
def test_static_results_1mod(method: str, update_static_results=False):
"""
Test if the results are constant. If the assertion fails, it means that the model or the evaluation has
changed, perhaps involuntarily.
"""
jsonfile = Path(__file__).parent / 'static_results.json'
static_results = json2dict(jsonfile)['static_results_1mod']
if method not in static_results:
write_to_jsonfile(jsonfile, [(f'static_results_1mod.{method}', {})])
static_results[method] = {}
static_results = static_results[method]
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname,
dataset='polymnist',
method=method,
attributes={
'num_flows': 0,
'num_mods': 1,
'deterministic': True,
'device': 'cpu',
'steps_per_training_epoch': 1,
'factorized_representation': False,
'calc_nll': False
})
trainer = PolymnistTrainer(mst)
test_results = trainer.run_epochs()
if update_static_results:
static_results['joint_div'] = test_results.joint_div
static_results['klds'] = test_results.klds['m0']
# static_results['lhoods'] = test_results.lhoods['m0']['m0']
static_results['log_probs'] = test_results.log_probs['m0']
static_results['total_loss'] = test_results.total_loss
# static_results['lr_eval'] = test_results.lr_eval['m0']['accuracy']
static_results['latents_class'] = {
'mu': test_results.latents['m0']['latents_class']['mu']
}
write_to_jsonfile(
jsonfile, [(f'static_results_1mod.{method}', static_results)])
are_they_equal = {
'joint_div':
np.round(test_results.joint_div,
5) == np.round(static_results['joint_div'], 5),
'klds':
np.round(test_results.klds['m0'],
5) == np.round(static_results['klds'], 5),
# 'lhoods': np.round(test_results.lhoods['m0']['m0'], 3) == np.round(static_results['lhoods'], 3),
'log_probs':
test_results.log_probs['m0'] == static_results['log_probs'],
'total_loss':
test_results.total_loss == static_results['total_loss'],
# 'lr_eval': test_results.lr_eval['m0']['accuracy'] == static_results['lr_eval'],
'latents_class_mu':
np.round(test_results.latents['m0']['latents_class']['mu'],
8) == np.round(static_results['latents_class']['mu'], 8)
}
assert all(
v for _, v in
are_they_equal.items()), f'Some results changed: {are_they_equal}'
def test_test_generation():
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname)
test_generation(mst)
def test_generate_plots():
with tempfile.TemporaryDirectory() as tmpdirname:
mst = set_me_up(tmpdirname)
generate_plots(mst, epoch=1)