def update_epoch(self, train_results: BaseBatchResults,
test_results: BaseTestResults, epoch: int,
epoch_time: float):
self.epoch_time.update(epoch_time)
# set beta_warmup coefficient
self.beta = min([
max([
self.flags.min_beta,
((epoch - self.flags.beta_start_epoch) *
(self.flags.max_beta - self.flags.min_beta)) /
max([self.flags.beta_warmup, 1.])
]), self.flags.max_beta
])
log.info(f'beta = {self.beta}')
self.maybe_send_to_db(train_results=train_results,
test_results=test_results,
epoch=epoch,
beta=self.beta)
# save checkpoints after every 5 epochs
if (epoch + 1) % self.flags.checkpoint_freq == 0 or (
epoch + 1) == self.flags.end_epoch:
self.exp.mm_vae.save_networks(epoch)
return self.beta
def compress_experiment_run_dir(flags) -> None:
"""
Move zipped experiment_dir_run in TMPDIR to experiment_dir.
"""
dir_experiment = Path(
json2dict(
get_config_path(flags=flags))['dir_experiment']).expanduser()
dir_experiment.mkdir(exist_ok=True)
# zip dir_experiment_run
log.info(
f'zipping {flags.dir_experiment_run} '
f'to {(Path(dir_experiment) / flags.experiment_uid).with_suffix(".zip")}.'
)
dir_experiment_zipped = (dir_experiment / flags.experiment_uid)
shutil.make_archive(dir_experiment_zipped,
'zip',
flags.dir_experiment_run,
verbose=True)
assert dir_experiment_zipped.with_suffix('.zip').exists(), f'{dir_experiment_zipped} does not exist. ' \
f'Zipping of dir_experiment_run failed.'
# delete not compressed experiment dir
shutil.rmtree(str(flags.dir_experiment_run))
def finalize(self, test_results: BaseTestResults, epoch: int, average_epoch_time):
log.info('Finalizing.')
# write results as json to experiment folder
run_metadata = {'end_epoch': epoch, 'experiment_duration': time.time() - self.begin_time,
'mean_epoch_time': self.callback.epoch_time.get_average()}
dict2json(self.flags.dir_experiment_run / 'results.json', test_results.__dict__ | run_metadata)
if self.flags.use_db == 1:
self.exp.experiments_database.insert_dict(run_metadata)
self.exp.experiments_database.save_networks_to_db(dir_checkpoints=self.flags.dir_checkpoints, epoch=epoch,
modalities=self.exp.mm_vae.modalities)
# self.exp.experiments_database.upload_logfile(self.flags.log_file)
self.exp.experiments_database.upload_tensorbardlogs(self.flags.dir_experiment_run / 'logs')
# run jupyter notebook with visualisations
pdf_path = run_notebook_convert(self.flags.dir_experiment_run)
# send alert
if self.flags.norby and self.flags.dataset != 'toy':
import ppb
import norby
expvis_url = ppb.upload(pdf_path, plain=True)
self.exp.experiments_database.insert_dict({'expvis_url': expvis_url})
norby.send_msg(f'Experiment {self.flags.experiment_uid} has finished. The experiment visualisation can be '
f'found here: {expvis_url}')
if self.flags.log_file.exists():
shutil.move(self.flags.log_file, self.flags.dir_experiment_run)
def __init__(self, dir_data: Path, transform=None, target_transform=None, num_modalities: int = 5):
"""
Args:
unimodal_datapaths (list): list of paths to weakly-supervised unimodal datasets with samples that
correspond by index. Therefore the numbers of samples of all datapaths should match.
transform: tranforms on colored MNIST digits.
target_transform: transforms on labels.
num_modalities (int): number of modalities.
"""
super().__init__()
if not dir_data.exists():
log.info(f'data dir {dir_data} does not exist. Creating PolyMNIST dataset. This may take a while...')
self.create_polymnist_dataset((dir_data / 'train'), Path(__file__).parent / 'polymnist_background_images',
num_modalities, train=True)
self.create_polymnist_dataset((dir_data / 'test'), Path(__file__).parent / 'polymnist_background_images',
num_modalities, train=False)
self.num_modalities = num_modalities
self.unimodal_datapaths = sorted(glob.glob(str(dir_data / 'm*')))[:num_modalities]
self.transform = transform
self.target_transform = target_transform
# save all paths to individual files
self.file_paths = {dp: [] for dp in self.unimodal_datapaths}
for dp in self.unimodal_datapaths:
assert Path(dp).exists(), f'data path {dp} does not exist.'
files = glob.glob(os.path.join(dp, "*.png"))
assert files, f'No png file found under {dp}'
self.file_paths[dp] = files
# assert that each modality has the same number of images
num_files = len(self.file_paths[dp])
for files in self.file_paths.values():
assert len(files) == num_files
self.num_files = num_files
def setup_leomed(self, flags):
tmpdir = Path(os.getenv("TMPDIR"))
# unzip polymnist dataset to tmpdir
if flags.dataset == 'polymnist':
polymnist_zip_path = Path(
'/cluster/work/vogtlab/Projects/Polymnist/PolyMNIST.zip')
out_dir = tmpdir
log.info(
f'Extracting data from {polymnist_zip_path} to {out_dir}.')
unpack_zipfile(polymnist_zip_path, out_dir)
flags.dir_data = out_dir / 'PolyMNIST'
assert out_dir.exists(), f'Data dir {out_dir} does not exist.'
# unzip celeba dataset to tmpdir
elif flags.dataset == 'celeba':
celeba_zip_path = Path(flags.dir_data).expanduser()
out_dir = tmpdir
log.info(f'Extracting data from {celeba_zip_path} to {out_dir}.')
unpack_zipfile(celeba_zip_path, out_dir)
flags.dir_data = out_dir / 'CelebA'
assert out_dir.exists(
), f'Data dir {out_dir} does not exist in {list(out_dir.parent.iterdir())}.'
flags.dir_fid = tmpdir / 'fid'
flags.dir_experiment = tmpdir
return flags
def delete_one(self, _id: str):
"""
Removes one document from db
"""
log.info(f'Deleting document with _id: {_id}.')
experiment = self.connect()
experiment.delete_one({'_id': _id})
def upload_one(exp_path: Path):
"""
Upload one experiment result to database together with the model checkpoints,
the logfile and tensorboardlogs, then delete zipped experiment dir.
"""
is_zip = exp_path.suffix == '.zip'
with tempfile.TemporaryDirectory() as tmpdirname:
tmpdir = Path(tmpdirname) / exp_path.stem
tmpdir.mkdir()
if is_zip:
# unpack zip into tmpdir
log.info(f'Unpacking {exp_path} to {tmpdir}.')
with zipfile.ZipFile(exp_path) as z:
z.extractall(tmpdir)
exp_dir = Path(tmpdir)
else:
exp_dir = exp_path
flags = torch.load(exp_dir / 'flags.rar')
db = MongoDatabase(training=True, flags=flags)
results = {'epoch_results': {}}
epochs = sorted(int(str(epoch.stem)) for epoch in (exp_dir / 'epoch_results').iterdir())
for epoch in epochs:
epoch_str = str(epoch)
epoch_results = (exp_dir / 'epoch_results' / epoch_str).with_suffix('.json')
results['epoch_results'][epoch_str] = json2dict(epoch_results)
db.insert_dict(results)
modalities = [mod_str for mod_str in results['epoch_results'][str(epoch)]['train_results']['log_probs'] if
len(mod_str.split('_')) == 1]
dir_checkpoints = exp_dir / 'checkpoints'
db.save_networks_to_db(
dir_checkpoints=dir_checkpoints,
epoch=max(int(str(d.name)) for d in dir_checkpoints.iterdir()),
modalities=modalities,
)
db.upload_tensorbardlogs(exp_dir / 'logs')
pdf_path = run_notebook_convert(exp_dir)
expvis_url = ppb.upload(pdf_path, plain=True)
db.insert_dict({'expvis_url': expvis_url})
log_file = glob.glob(str(exp_dir) + '/*.log')
if len(log_file):
db.upload_logfile(Path(log_file[0]))
send_msg(f'Uploading of experiment {flags.experiment_uid} has finished. The experiment visualisation can be '
f'found here: {expvis_url}'
)
# delete exp_path
if is_zip:
exp_path.unlink()
else:
shutil.rmtree(exp_path)
def get_hyperopt_score(test_results: BaseTestResults, use_zk: bool,
optuna_trial):
"""
Sum over all metrics to get a score for the optimization of hyperparameters.
"""
# score_gen_eval = np.mean([score for _, score in test_results.gen_eval.items()])
score_gen_eval = np.mean([
score
for score in get_missing_mod_scores_gen_eval(test_results.gen_eval)
])
# score_prd = np.mean([score for _, score in test_results.prd_scores.items()])
score_prd = np.mean([
score for score in get_missing_mod_scores_prd(test_results.prd_scores)
])
# if use_zk:
# score_lr_eval = np.mean([score['accuracy'] for _, score in test_results.lr_eval_zk.items()])
# else:
# score_lr_eval = np.mean([score['accuracy'] for _, score in test_results.lr_eval_q0.items()])
# add metrics to optuna so that they can be retrieved in the database.
optuna_trial.suggest_float("score_gen_eval", score_gen_eval,
score_gen_eval)
optuna_trial.suggest_float("score_prd", score_prd, score_prd)
# optuna_trial.suggest_float("score_lr_eval", score_lr_eval, score_lr_eval)
# score = score_lr_eval / 0.9 + score_gen_eval / 0.55 + score_prd / 0.05
score = score_gen_eval / 0.55 + score_prd / 0.05
# score = score_lr_eval / 0.75 + score_gen_eval / 0.55 + score_prd / 0.1
log.info(f'Current hyperopt score is {score}. '
# f'score_lr_eval: {score_lr_eval}, '
f'score_gen_eval: {score_gen_eval}, '
f'score_prd: {score_prd}')
return optuna_trial, score
def run_hyperopt_epochs(trainer: PolymnistTrainer) -> int:
test_results = trainer.run_epochs()
# clean experiment run dir
shutil.rmtree(trainer.flags.dir_experiment_run)
log.info(
f'Finished hyperopt run with score: {test_results.hyperopt_score}.')
return test_results.hyperopt_score
def stdout_if_verbose(verbose: int, message, min_level: int):
"""
verbose: current global verbose setting
message: message to be sent to stdout
level: minimum verbose level needed to send the message
"""
if verbose >= min_level:
log.info(message)
def run_one(exp_dir: str):
exp_dir = Path(exp_dir).expanduser()
log.info(
f'Starting execution of experiment vis for experiment {exp_dir.name}')
pdf_path = run_notebook_convert(exp_dir)
expvis_url = ppb.upload(pdf_path, plain=True)
log.info(f'Uploaded experiment vis to {expvis_url}')
db = MongoDatabase(training=False, _id=exp_dir.name)
db.insert_dict({'expvis_url': expvis_url})
def get_ds_stats(args):
path = Path(__file__).parent.parent / 'data/dataset_stats.json'
if not path.exists():
from mimic.dataio.find_dataset_stats import get_mean_std
log.info(
"Dataset statistics not found, need to create them. This may take a while."
)
get_mean_std(out_path=path, args=args)
with open(path, 'r') as jsonfile:
data = json.load(jsonfile)
return data['PA_mean'], data['PA_std']
def set_rec_weights(self):
"""
Sets the weights of the log probs for each modality.
"""
log.info('setting rec_weights')
return {
'PA': self.flags.rec_weight_m1,
'Lateral': self.flags.rec_weight_m2,
'text': self.flags.rec_weight_m3
}
def set_dataset(self):
log.info('setting dataset')
# used for faster unittests i.e. a dummy dataset
if self.flags.dataset == 'toy':
log.info('using testing dataset')
self.flags.vocab_size = 3517
d_train = Mimic_testing(self.flags)
d_eval = Mimic_testing(self.flags)
else:
d_train = Mimic(self.flags, self.labels, split='train')
d_eval = Mimic(self.flags, self.labels, split='eval')
return d_train, d_eval
def load_networks_from_db(self, mmvae: BaseMMVae):
log.info(f'Loading networks from database for model {mmvae}.')
fs = self.connect_with_gridfs()
fs_ids = [elem._id for elem in fs.find({})]
with tempfile.TemporaryDirectory() as tmpdirname:
tmpdirname = Path(tmpdirname)
for mod_str in mmvae.modalities:
for prefix in ['en', 'de']:
filename = tmpdirname / f"{prefix}coderM{mod_str}"
model_id = self.experiment_uid + f"__{prefix}coderM{mod_str}"
with open(filename, 'wb') as fileobject:
fileobject.write(fs.get(model_id).read())
mmvae.load_networks(tmpdirname)
return mmvae
def save_networks_to_db(self, dir_checkpoints: Path, epoch: int,
modalities):
"""
Inspired from https://medium.com/naukri-engineering/way-to-store-large-deep-learning-models-in-production-ready-environments-d8a4c66cc04c
There is probably a better way to store Tensors in MongoDB.
"""
fs = self.connect_with_gridfs()
checkpoint_dir = dir_checkpoints / str(epoch).zfill(4)
fs_ids = [elem._id for elem in fs.find({})]
for mod_str in modalities:
for prefix in ['en', 'de']:
filename = checkpoint_dir / f"{prefix}coderM{mod_str}"
_id = self.experiment_uid + f"__{prefix}coderM{mod_str}"
if _id not in fs_ids:
with io.FileIO(str(filename), 'r') as fileObject:
log.info(f'Saving checkpoint to db: {filename}')
fs.put(fileObject, filename=str(filename), _id=_id)
def upload_notebook_to_db(experiment_uid: str) -> None:
"""
Run the experiment vis notebook and upload it with ppb to db.
"""
import ppb
with tempfile.TemporaryDirectory() as tmpdirname:
dir_experiment_run = Path(tmpdirname) / experiment_uid
dir_experiment_run.mkdir()
db = MongoDatabase(training=False, _id=experiment_uid)
dict2json(dir_experiment_run / 'flags.json', db.get_experiment_dict()['flags'])
pdf_path = run_notebook_convert(dir_experiment_run=dir_experiment_run)
expvis_url = ppb.upload(pdf_path, plain=True)
log.info(f'Experiment_vis was uploaded to {expvis_url}')
db.insert_dict({'expvis_url': expvis_url})
def __init__(self, args, alphabet, partition=0, transform=None):
self.dir_dataset_base = args.dir_data
if not self.dir_dataset_base.exists():
log.info(
f'Dataset not found under {self.dir_dataset_base}. Parent directory contains: '
f'{list(self.dir_dataset_base.parent)}')
download_zip_from_url(
url=
'https://www.dropbox.com/sh/lx8669lyok9ois6/AACCZqDiZuv0Q8RA3Qmwrwnca/celeba_data.zip?dl=1',
dest_folder=Path(self.dir_dataset_base).parent,
verbose=True)
filename_text = self.dir_dataset_base / (
'list_attr_text_' + str(args.len_sequence).zfill(3) + '_' +
str(args.random_text_ordering) + '_' +
str(args.random_text_startindex) + '_celeba.csv')
filename_partition = os.path.join(self.dir_dataset_base,
'list_eval_partition.csv')
filename_attributes = os.path.join(self.dir_dataset_base,
'list_attr_celeba.csv')
df_text = pd.read_csv(filename_text)
df_partition = pd.read_csv(filename_partition)
df_attributes = pd.read_csv(filename_attributes)
self.args = args
self.img_dir = os.path.join(self.dir_dataset_base, 'img_align_celeba')
self.txt_path = filename_text
self.attrributes_path = filename_attributes
self.partition_path = filename_partition
self.alphabet = alphabet
self.img_names = df_text.loc[df_partition['partition'] ==
partition]['image_id'].values
self.attributes = df_attributes.loc[df_partition['partition'] ==
partition]
self.labels = df_attributes.loc[
df_partition['partition'] ==
partition].values # atm, i am just using blond_hair as labels
self.y = df_text.loc[df_partition['partition'] ==
partition]['text'].values
self.transform = transform
def classify_cond_gen_samples(exp, labels: Tensor, cond_samples: typing.Mapping[str, Tensor]) \
-> typing.Mapping[str, Tensor]:
"""
Classifies for each modality all the conditionally generated samples.
Returns a dict like the following:
{'PA': tensor,
'Lateral': tensor,
'text': tensor}
"""
clf_predictions = {mod: {} for mod in exp.modalities}
for mod in exp.modalities:
if mod in cond_samples:
mod_cond_gen: Tensor = cond_samples[mod]
mod_clf = exp.modalities[mod].clf
# classify generated sample to evaluate coherence
clf_predictions[mod] = mod_clf(mod_cond_gen).cpu()
else:
log.info(str(mod) + 'not existing in cond_gen_samples')
return clf_predictions
def set_modalities(self) -> typing.Mapping[str, BaseModality]:
log.info('setting modalities')
mods = {}
for mod_str in self.flags.mods.split('_'):
if mod_str == 'F':
mod = MimicPA(self.flags, self.labels,
self.flags.rec_weight_m1, self.plot_img_size)
elif mod_str == 'L':
mod = MimicLateral(self.flags, self.labels,
self.flags.rec_weight_m2,
self.plot_img_size)
elif mod_str == 'T':
mod = MimicText(self.flags, self.labels,
self.flags.rec_weight_m3, self.plot_img_size,
self.dataset_train.report_findings_dataset.i2w)
else:
raise ValueError(f'Invalid mod_str {mod_str}.' +
'Choose between {T,L,T}')
mods[mod.name] = mod
return mods
def upload_tensorbardlogs(self, tensorboard_logdir: Path) -> None:
"""zip tensorboard logs and save them to db."""
fs = self.connect_with_gridfs()
fs_ids = [elem._id for elem in fs.find({})]
file_id = self.experiment_uid + f"__tensorboard_logs"
if file_id not in fs_ids:
with tempfile.TemporaryDirectory() as tmpdirname:
log.info(f'Zipping {file_id} to {tmpdirname}.')
zipfile = Path(tmpdirname) / tensorboard_logdir.name
shutil.make_archive(zipfile,
'zip',
tensorboard_logdir,
verbose=True)
log.info(f'Uploading tensorboard logs to db.')
with io.FileIO(str(zipfile.with_suffix('.zip')),
'r') as fileObject:
fs.put(fileObject,
filename=str(tensorboard_logdir.name),
_id=file_id)
def get_lr_training_data(args, exp, mm_vae, subsets: List[str], train_loader,
num_training_samples_lr):
"""
Get the latent embedding as the training data for the linear classifiers.
"""
data_train = None
all_labels = torch.Tensor()
log.info(
f"Creating {num_training_samples_lr} batches of the latent representations for the classifier."
)
for it, (batch_d, batch_l) in tqdm(enumerate(train_loader),
total=len(train_loader),
postfix='creating_train_lr'):
"""
Constructs the training set (labels and inferred subsets) for the classifier training.
"""
if it >= 1000 and it >= num_training_samples_lr:
break
batch_d = {k: v.to(exp.flags.device) for k, v in batch_d.items()}
_, joint_latent = mm_vae.module.inference(
batch_d) if args.distributed else mm_vae.inference(batch_d)
joint_latent: Union[JointLatents, JointLatentsFoEM, JointLatentsFoJ,
JointLatentsFoS]
all_labels = torch.cat((all_labels, batch_l), 0)
lr_data = joint_latent.get_lreval_data()
if data_train is None:
data_train = lr_data
else:
for which_lr in lr_data:
for s_key in lr_data[which_lr]:
data_train[which_lr][s_key] = torch.cat(
(data_train[which_lr][s_key],
lr_data[which_lr][s_key]), 0)
return all_labels, data_train
def run_notebook_convert(dir_experiment_run: Path = None) -> Path:
"""
Run and convert the notebook to html and pdf.
"""
# Copy the experiment_vis jupyter notebook to the experiment dir
notebook_path = Path(__file__).parent.parent / 'experiment_vis/experiment_vis.ipynb'
dest_notebook_path = dir_experiment_run / 'experiment_vis.ipynb'
# copy notebook to experiment run
shutil.copyfile(notebook_path, dest_notebook_path)
log.info(f'Executing experiment vis notebook {dest_notebook_path}.')
with open(dest_notebook_path) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': str(dest_notebook_path.parent)}})
nbconvert_path = dest_notebook_path.with_suffix('.nbconvert.ipynb')
with open(nbconvert_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
log.info('Converting notebook to html.')
html_path = nbconvert_path.with_suffix('.html')
html_exporter = HTMLExporter()
html_exporter.template_name = 'classic'
(body, resources) = html_exporter.from_notebook_node(nb)
with open(html_path, 'w') as f:
f.write(body)
log.info('Converting notebook to pdf.')
pdf_path = nbconvert_path.with_suffix('.pdf')
pdf_exporter = PDFExporter()
pdf_exporter.template_name = 'classic'
(body, resources) = pdf_exporter.from_notebook_node(nb)
pdf_path.write_bytes(body)
return pdf_path
from mmvae_hub import log
from mmvae_hub.celeba.CelebaTrainer import CelebaTrainer
from mmvae_hub.celeba.experiment import CelebaExperiment
from mmvae_hub.celeba.flags import parser, CelebaFlagsSetup
from mmvae_hub.leomed_utils.boilerplate import compress_experiment_run_dir
from mmvae_hub.utils.setup.flags_utils import get_config_path
DATASET = 'celeba'
if __name__ == '__main__':
flags = parser.parse_args()
flags_setup = CelebaFlagsSetup(
get_config_path(dataset=DATASET, flags=flags))
flags = flags_setup.setup(flags, additional_args={'dataset': DATASET})
with maybe_norby(flags.norby,
f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
mst = CelebaExperiment(flags)
mst.set_optimizer()
trainer = CelebaTrainer(mst)
trainer.run_epochs()
log.info('Done.')
# move zipped experiment_dir_run in TMPDIR to experiment_dir
if flags.leomed:
compress_experiment_run_dir(flags)
def insert_dict(self, d: dict):
log.info('Inserting dict to database.')
experiments = self.connect()
experiments.find_one_and_update({'_id': self.experiment_uid},
{"$set": d})
def test(self, epoch, last_epoch: bool) -> BaseTestResults:
with torch.no_grad():
self.exp.set_eval_mode()
model = self.exp.mm_vae
d_loader, training_steps, average_meters = self.setup_phase('test')
for iteration, (batch_d, _) in enumerate(at_most_n(d_loader, training_steps)):
batch_d = model.batch_to_device(batch_d)
forward_results: BaseForwardResults = model(batch_d)
# calculate the loss
total_loss, joint_divergence, log_probs, klds = model.calculate_loss(forward_results, batch_d)
results = {**forward_results.__dict__, 'joint_divergence': joint_divergence}
batch_results = {
'total_loss': total_loss.item(),
'klds': get_items_from_dict(klds),
'log_probs': get_items_from_dict(log_probs),
'joint_divergence': results['joint_divergence'].item(),
# 'latents': forward_results.enc_mods,
# 'joint_latents': forward_results.joint_latents
}
for key in batch_results:
average_meters[key].update(batch_results[key])
averages = {k: v.get_average() for k, v in average_meters.items()}
self.tb_logger.write_testing_logs(**{k: v for k, v in averages.items() if k != 'joint_latents'})
test_results = BaseTestResults(joint_div=averages['joint_divergence'], **averages)
log.info('generating plots')
# temp
# plots = generate_plots(self.exp, epoch)
# self.tb_logger.write_plots(plots, epoch)
if self.flags.eval_lr:
log.info('evaluation of latent representation')
# train linear classifiers
clf_lr_q0, clf_lr_zk = train_clf_lr_all_subsets(self.exp)
# test linear classifiers
# methods where the flow is applied on each modality don't have a q0.
lr_eval_q0 = test_clf_lr_all_subsets(clf_lr_q0, self.exp, which_lr='q0') \
if clf_lr_q0 else None
lr_eval_zk = test_clf_lr_all_subsets(clf_lr_zk, self.exp, which_lr='zk') \
if clf_lr_zk else None
# log results
lr_eval_results = {'q0': lr_eval_q0, 'zk': lr_eval_zk}
log.info(f'Lr eval results: {lr_eval_results}')
self.tb_logger.write_lr_eval(lr_eval_results)
test_results.lr_eval_q0 = lr_eval_q0
test_results.lr_eval_zk = lr_eval_zk
if self.flags.use_clf:
log.info('test generation')
gen_eval = test_generation(self.exp)
log.info(f'Gen eval results: {gen_eval}')
self.tb_logger.write_coherence_logs(gen_eval)
test_results.gen_eval = flatten_cond_gen_values(gen_eval)
if self.flags.calc_nll:
log.info('estimating likelihoods')
lhoods = estimate_likelihoods(self.exp)
self.tb_logger.write_lhood_logs(lhoods)
test_results.lhoods = lhoods
if self.flags.calc_prd and (((epoch + 1) % self.flags.eval_freq_fid == 0) or last_epoch):
log.info('calculating prediction score')
prd_scores = calc_prd_score(self.exp)
self.tb_logger.write_prd_scores(prd_scores)
test_results.prd_scores = prd_scores
return test_results
def test_clf_lr_all_subsets(clf_lr: Mapping[str, Mapping[str,
LogisticRegression]],
exp, which_lr: str):
"""
Test the classifiers that were trained on latent representations.
which_lr: either q0.mu or zk.
"""
args = exp.flags
mm_vae = exp.mm_vae
mm_vae.eval()
d_loader = DataLoader(exp.dataset_test,
batch_size=exp.flags.batch_size,
shuffle=False,
num_workers=exp.flags.dataloader_workers,
drop_last=True)
training_steps = exp.flags.steps_per_training_epoch or len(d_loader)
log.info(
f'Creating {training_steps} batches of latent representations for classifier testing '
f'with a batch_size of {exp.flags.batch_size}.')
# clf_predictions = init_clf_predictions(subsets, which_lr, mm_vae)
clf_predictions = {}
batch_labels = torch.Tensor()
for iteration, (batch_d, batch_l) in enumerate(d_loader):
if iteration > training_steps:
break
batch_labels = torch.cat((batch_labels, batch_l), 0)
batch_d = dict_to_device(batch_d, exp.flags.device)
_, joint_latent = mm_vae.module.inference(
batch_d) if args.distributed else mm_vae.inference(batch_d)
lr_subsets = joint_latent.subsets
lr_data = joint_latent.get_lreval_data()
data_test = lr_data[which_lr]
clf_predictions_batch = classify_latent_representations(
exp, clf_lr, data_test)
clf_predictions_batch: Mapping[str, Mapping[str, np.array]]
for subset in data_test:
clf_predictions_batch_subset = torch.cat(
tuple(
torch.tensor(clf_predictions_batch[label]
[subset]).unsqueeze(1)
for label in exp.labels), 1)
if subset in clf_predictions:
clf_predictions[subset] = torch.cat(
[clf_predictions[subset], clf_predictions_batch_subset], 0)
else:
clf_predictions[subset] = clf_predictions_batch_subset
batch_labels = atleast_2d(batch_labels, -1)
results = {}
for subset in clf_predictions:
# calculate metrics
metrics = exp.metrics(clf_predictions[subset],
batch_labels,
str_labels=exp.labels)
metrics_dict = metrics.evaluate()
results[subset] = metrics.extract_values(metrics_dict)
return results
def create_dir_structure(flags: argparse.ArgumentParser(),
train: bool = True) -> argparse.ArgumentParser:
"""
Create following directories under flags.dir_experiment:
experiment_uid, checkpoints, logs, generation_evaluation, inference, fid_eval, plots.
Under flags.dir_experiment/plots, create:
cond_gen, random_samples, swapping
"""
experiment_uid = flags.experiment_uid
if train:
flags.dir_experiment_run = flags.dir_experiment / experiment_uid
flags.str_experiment = experiment_uid
else:
flags.dir_experiment_run = (flags.dir_experiment).expanduser()
log.info(f'dir_experiment_run: {flags.dir_experiment_run}')
if train:
create_dir(flags.dir_experiment_run)
flags.dir_checkpoints = Path(flags.dir_experiment_run) / 'checkpoints'
if train:
create_dir(os.path.expanduser(flags.dir_checkpoints))
flags.dir_logs = os.path.join(os.path.expanduser(flags.dir_experiment_run),
'logs')
if train:
create_dir(flags.dir_logs)
flags.dir_gen_eval = os.path.join(
os.path.expanduser(flags.dir_experiment_run), 'generation_evaluation')
if train:
create_dir(flags.dir_gen_eval)
flags.dir_inference = os.path.join(
os.path.expanduser(flags.dir_experiment_run), 'inference')
if train:
create_dir(flags.dir_inference)
if not train:
flags.dir_fid = os.path.join(flags.dir_experiment_run, 'fid_eval')
if not os.path.exists(flags.dir_fid):
os.makedirs(flags.dir_fid)
flags.dir_gen_eval_fid = os.path.join(flags.dir_fid, experiment_uid)
log.info(f'dir_gen_eval_fid: {flags.dir_gen_eval_fid}')
create_dir(flags.dir_gen_eval_fid)
flags.dir_plots = os.path.join(flags.dir_experiment_run, 'plots')
if train:
create_dir(flags.dir_plots)
flags.dir_swapping = os.path.join(flags.dir_plots, 'swapping')
if train:
create_dir(flags.dir_swapping)
flags.dir_random_samples = os.path.join(flags.dir_plots, 'random_samples')
if train:
create_dir(flags.dir_random_samples)
flags.dir_cond_gen = os.path.join(flags.dir_plots, 'cond_gen')
if train:
create_dir(flags.dir_cond_gen)
return flags
