def df_maker(epoch: int):
config = json2dict(Path(('conf.json')))
methods = config['methods']
data_dir = Path(__file__).parent.parent / 'data/thesis'
experiment_uids_path = data_dir / ('experiment_uids.json')
exp_uids = json2dict(experiment_uids_path)['polymnist']
for method in methods:
method_uids = exp_uids[method]['3_mods']
d = {}
for method_uid in method_uids:
epoch_results_dir = data_dir / 'experiments' / 'polymnist' / method / method_uid / 'epoch_results'
lr_eval_results = {
k: np.round(v['accuracy'], 3)
for k, v in json2dict(epoch_results_dir / f'{epoch}.json')
['test_results']['lr_eval_q0'].items()
}
for k, v in lr_eval_results.items():
if k != 'joint':
if k not in d:
d[k] = [v]
else:
d[k].append(v)
d_ = {}
for k, v in d.items():
d_[k] = np.round(np.mean(v), 3)
d_[k + '__STDEV'] = np.round(np.std(v), 3)
yield {
'Method': method,
**d_,
}
def df_maker(epoch: int, methods):
data_dir = Path(__file__).parent.parent / 'data/thesis'
experiment_uids_path = data_dir / ('experiment_uids.json')
exp_uids = json2dict(experiment_uids_path)['polymnist']
for method in methods:
method_uids = exp_uids[method]['3_mods']
d = {
'missing_mod_scores': [],
'reconstr_mod_scores': [],
'random_gen_scores': []
}
for method_uid in method_uids:
epoch_results_dir = data_dir / 'experiments' / 'polymnist' / method / method_uid / 'epoch_results'
gen_eval_dict = flatten_dict(
json2dict(epoch_results_dir /
f'{epoch}.json')['test_results']['gen_eval'])
d['random_gen_scores'].append(
np.mean([
score for k, score in gen_eval_dict.items()
if k.startswith('random')
]))
gen_eval_dict = {
k: v
for k, v in gen_eval_dict.items() if not k.startswith('random')
}
d['missing_mod_scores'].append(
np.mean([
score
for score in get_missing_mod_scores_gen_eval(gen_eval_dict)
]))
d['reconstr_mod_scores'].append(
np.mean([
score for score in get_reconstr_mod_scores_gen_eval(
gen_eval_dict)
]))
yield {
'Method': method,
'Missing Mod': np.round(np.mean(d['missing_mod_scores']), 3),
'Reconstruction': np.round(np.mean(d['reconstr_mod_scores']), 3),
'Random': np.round(np.mean(d['random_gen_scores']), 3),
'Missing Mod__STDEV': np.round(np.std(d['missing_mod_scores']), 3),
'Reconstruction__STDEV': np.round(np.std(d['reconstr_mod_scores']),
3),
'Random__STDEV': np.round(np.std(d['random_gen_scores']), 3),
}
def df_maker():
for method in exp_uids['iw_comparison']:
d = {}
for K in [1, 3, 5]:
if K == 1:
method_ = method.replace('iw', '')
uid = exp_uids['polymnist'][method_]['3_mods'][0]
exp_dir = data_dir / f'experiments/polymnist/{method_}/{uid}'
else:
uid = exp_uids['iw_comparison'][method][f"K{K}"][0]
exp_dir = data_dir / f"experiments/iw_comp/{method}/{uid}"
res_dict = json2dict(
exp_dir /
f"epoch_results/{conf['max_epoch']['polymnist'] - 1}.json")
d[f'lr_{K}'] = np.mean([
v['accuracy']
for _, v in res_dict['test_results']['lr_eval_q0'].items()
])
d[f'coherence_{K}'] = np.mean([
v for _, v in flatten_dict(res_dict['test_results']
['gen_eval']).items()
])
d[f'prd_{K}'] = np.mean([
v for _, v in flatten_dict(res_dict['test_results']
['prd_scores']).items()
])
d['method'] = method
yield d
return {_class: i for i, _class in enumerate(self.classes)}
def _check_exists_mnist(self):
return (Path(self.processed_folder) / self.training_file_mnist).exists() and (
Path(self.processed_folder) / self.test_file_mnist).exists()
def _check_exists_svhn(self):
return (os.path.exists(os.path.join(self.dir_svhn,
self.training_file_svhn)) and
os.path.exists(os.path.join(self.dir_svhn,
self.test_file_svhn)))
@staticmethod
def extract_gzip(gzip_path, remove_finished=False):
print('Extracting {}'.format(gzip_path))
with open(gzip_path.replace('.gz', ''), 'wb') as out_f, \
gzip.GzipFile(gzip_path) as zip_f:
out_f.write(zip_f.read())
if remove_finished:
os.unlink(gzip_path)
def extra_repr(self):
return "Split: {}".format("Train" if self.train is True else "Test")
if __name__ == '__main__':
config = json2dict(Path(get_config_path(dataset='mnistsvhntext')))
download_zip_from_url(
url='https://www.dropbox.com/sh/lx8669lyok9ois6/AADMhr3EluBXJyZnV1_lYntTa/data_mnistsvhntext.zip?dl=1',
dest_folder=Path(config['dir_data']).expanduser().parent, verbose=True)
from pathlib import Path
from shutil import rmtree
from modun.file_io import json2dict
data_dir = Path(__file__).parent.parent.parent / 'data'
exp_uids = json2dict(data_dir / 'thesis/experiment_uids.json')
experiments_dir = data_dir / 'thesis/experiments'
for dataset in ['polymnist', 'mimic']:
for method in exp_uids[dataset]:
for num_mods in exp_uids[dataset][method]:
uids = exp_uids[dataset][method][num_mods]
# for uid in uids:
# if not (experiments_dir / dataset / method / uid).exists():
# print(num_mods)
# print(uid)
new_uids = [
uid for uid in uids
if (experiments_dir / dataset / method / uid).exists()
]
if not new_uids:
new_uids = [""]
exp_uids[dataset][method][num_mods] = new_uids
# dict2json(d=exp_uids, out_path=data_dir / 'thesis/experiment_uids.json')
# clean experiment_dirs
for dataset in ['polymnist', 'mimic']:
for method_dir in (experiments_dir / dataset).iterdir():
import random
from pathlib import Path
import torch
from matplotlib import pyplot as plt
from mmvae_hub.experiment_vis.utils import load_experiment
from mmvae_hub.utils.plotting import plot
from modun.file_io import json2dict
data_dir = Path(__file__).parent.parent / 'data/thesis/polymnist'
exp_uids = json2dict(data_dir / 'experiment_uids.json')
num_samples = 30
config = json2dict(Path('conf.json'))
# for method in config['methods']:
for method in ['iwmogfm']:
save_path = data_dir / f'{method}/rand_gen_plot.png'
_id = exp_uids[method]['3_mods'][0]
experiment_dir = data_dir / _id
exp = load_experiment(experiment_dir=experiment_dir, _id=_id)
model = exp.mm_vae
mods = exp.modalities
random_samples = model.generate(num_samples)
m_keys = list(mods.keys())
rec = torch.zeros(mods['m0'].plot_img_size,
dtype=torch.float32).repeat(num_samples, 1, 1, 1)
from pathlib import Path
from modun.file_io import json2dict
from lib.utils import tex_escape
from scripts_ import tikz
conf = json2dict(Path('prepare_thesis/conf.json'))
methods = conf['methods']
pic = tikz.Picture()
yshift = 0
for method in methods:
img_path = Path(f'data/thesis/polymnist/{method}/rand_gen_plot.png')
if method == 'iwmogfm_amortized':
pic.set_node(text=fr'\Large{{\textbf{{iwmogfm}}}}\\\Large{{\textbf{{amortized}}}}',
options=f'yshift=-{yshift}cm, align=center', name=method)
else:
pic.set_node(text=fr'\Large{{\textbf{{{tex_escape(method)}}}}}', options=f'yshift=-{yshift}cm', name=method)
# pic.set_node(text=fr'\Large{{\textbf{{{tex_escape(method)}}}}}', options=f'yshift=-{yshift}cm', name=method)
pic.set_node(text=f'\\includegraphics[scale=0.8]{{{img_path}}}', options=f'right of={method}, xshift=8.5cm')
yshift += 6
print(pic.make())
lr_eval_results = {
k: np.round(v['accuracy'], 3)
for k, v in json2dict(epoch_results_dir / f'{epoch}.json')
['test_results']['lr_eval_q0'].items()
}
for k, v in lr_eval_results.items():
if k != 'joint':
if k not in d:
d[k] = [v]
else:
d[k].append(v)
d_ = {}
for k, v in d.items():
d_[k] = np.round(np.mean(v), 3)
d_[k + '__STDEV'] = np.round(np.std(v), 3)
yield {
'Method': method,
**d_,
}
if __name__ == '__main__':
config = json2dict(Path('conf.json'))
lr_eval_df = pd.DataFrame(data=df_maker(
epoch=config['max_epoch']['polymnist'] - 1))
lr_eval_df.to_csv(Path(__file__).parent.parent / 'data/thesis/lr_eval.csv',
index=False)
score
for score in get_missing_mod_scores_gen_eval(gen_eval_dict)
]))
d['reconstr_mod_scores'].append(
np.mean([
score for score in get_reconstr_mod_scores_gen_eval(
gen_eval_dict)
]))
yield {
'Method': method,
'Missing Mod': np.round(np.mean(d['missing_mod_scores']), 3),
'Reconstruction': np.round(np.mean(d['reconstr_mod_scores']), 3),
'Random': np.round(np.mean(d['random_gen_scores']), 3),
'Missing Mod__STDEV': np.round(np.std(d['missing_mod_scores']), 3),
'Reconstruction__STDEV': np.round(np.std(d['reconstr_mod_scores']),
3),
'Random__STDEV': np.round(np.std(d['random_gen_scores']), 3),
}
if __name__ == '__main__':
config = json2dict(Path(__file__).parent / 'conf.json')
gen_eval_df = pd.DataFrame(data=df_maker(
epoch=config['max_epoch']['polymnist'] - 1, methods=config['methods']))
gen_eval_df.to_csv(Path(__file__).parent.parent /
'data/thesis/gen_eval.csv',
index=False)
label = torch.from_numpy(
(self.labels[index, 1:] > 0).astype(int)).float()
# img = torch.rand((self.args.image_channels, 64, 64))
# text_str = torch.ones((8, 71))
# img = torch.ones((1, 28, 28))
# text_str = torch.rand((256, 71))
sample = {'img': img, 'text': text_str}
# sample = {'img': img}
return sample, label
def __len__(self):
return self.y.shape[0]
def get_text_str(self, index):
return self.y[index]
if __name__ == '__main__':
config = json2dict(get_config_path(dataset='celeba'))
dir_data = Path(config['dir_data']).expanduser()
if not dir_data.exists():
download_zip_from_url(
url=
'https://www.dropbox.com/sh/lx8669lyok9ois6/AACCZqDiZuv0Q8RA3Qmwrwnca/celeba_data.zip?dl=1',
dest_folder=dir_data.parent,
verbose=True)
print("Done.")
def df_maker(epoch: int):
config = json2dict(Path(('conf.json')))
methods = config['methods']
data_dir = Path(__file__).parent.parent / 'data/thesis'
experiment_uids_path = data_dir / ('experiment_uids.json')
exp_uids = json2dict(experiment_uids_path)['polymnist']
for method in methods:
method_uids = exp_uids[method]['3_mods']
d = {
'missing_mod_scores': [],
'reconstr_mod_scores': [],
'random_prd_scores': []
}
for method_uid in method_uids:
epoch_results_dir = data_dir / 'experiments' / 'polymnist' / method / method_uid / 'epoch_results'
prd_dict = json2dict(epoch_results_dir /
f'{epoch}.json')['test_results']['prd_scores']
if prd_dict is None:
tmpdirname = Path('/mnt/data/hendrik/mmnf_data/tempdir')
tmpdirname.mkdir()
experiment_dir = data_dir / 'experiments' / method / method_uid
exp = load_experiment(
experiment_dir,
_id=method_uid,
epoch=epoch,
add_args={'dir_gen_eval_fid': tmpdirname})
args = exp.flags
mm_vae = exp.mm_vae
rand_gen = mm_vae.generate()
d_loader = DataLoader(exp.dataset_test,
batch_size=args.batch_size,
shuffle=True,
num_workers=exp.flags.dataloader_workers,
drop_last=True)
for iteration, (batch_d, batch_l) in tqdm(enumerate(d_loader),
total=len(d_loader)):
batch_d = dict_to_device(batch_d, exp.flags.device)
save_generated_samples(exp, rand_gen, iteration, batch_d)
_, joint_latent = mm_vae.inference(batch_d)
cg = mm_vae.cond_generation(joint_latent)
for subset, cond_val in cg.items():
save_generated_samples_singlegroup(
exp, iteration, subset, cond_val)
prd_dict = calc_prd_score(exp)
ep_res_dict = json2dict(epoch_results_dir / f'{epoch}.json')
ep_res_dict['test_results']['prd_scores'] = prd_dict
dict2json(out_path=epoch_results_dir / f'{epoch}.json',
d=ep_res_dict)
tmpdirname.rmdir()
d['random_prd_scores'].append(
np.mean([
score for k, score in prd_dict.items()
if k.startswith('random')
]))
prd_dict = {
k: v
for k, v in prd_dict.items() if not k.startswith('random')
}
d['missing_mod_scores'].append(
np.mean(
[score for score in get_missing_mod_scores_prd(prd_dict)]))
d['reconstr_mod_scores'].append(
np.mean([
score for score in get_reconstr_mod_scores_prd(prd_dict)
]))
yield {
'Method': method,
'Missing Mod': np.round(np.mean(d['missing_mod_scores']), 3),
'Reconstruction': np.round(np.mean(d['reconstr_mod_scores']), 3),
'Random': np.round(np.mean(d['random_prd_scores']), 3),
'Missing Mod__STDEV': np.round(np.std(d['missing_mod_scores']), 3),
'Reconstruction__STDEV': np.round(np.std(d['reconstr_mod_scores']),
3),
'Random__STDEV': np.round(np.std(d['random_prd_scores']), 3),
}
