def load_experiment(experiment_dir: Path = None, flags=None, _id: str = None, epoch=None, add_args=None):
"""Load experiment with flags and trained model from old experiment."""
if add_args is None:
add_args = {}
if not flags:
dataset = _id.split('_')[0].lower()
flags_setup = BaseFlagsSetup(get_config_path(dataset=dataset))
if experiment_dir is not None:
flags_path = experiment_dir / Path('flags.rar')
if not experiment_dir is None and flags_path.exists():
flags = flags_setup.load_old_flags(flags_path, add_args={'save_figure': False, **add_args})
else:
flags = flags_setup.load_old_flags(_id=_id, add_args={'save_figure': False, **add_args})
exp = get_experiment(flags)
if experiment_dir and (experiment_dir / 'checkpoints').exists():
latest_checkpoint = epoch if epoch \
else max(int(d.name) for d in (experiment_dir / 'checkpoints').iterdir() if d.name.isdigit())
print(f'loading checkpoint from epoch {latest_checkpoint}.')
latest_checkpoint_path = experiment_dir / 'checkpoints' / str(latest_checkpoint).zfill(4)
exp.mm_vae.load_networks(latest_checkpoint_path)
else:
# load networks from database
exp.mm_vae = exp.experiments_database.load_networks_from_db(exp.mm_vae)
return exp
def compress_experiment_run_dir(flags) -> None:
"""
Move zipped experiment_dir_run in TMPDIR to experiment_dir.
"""
dir_experiment = Path(
json2dict(
get_config_path(flags=flags))['dir_experiment']).expanduser()
dir_experiment.mkdir(exist_ok=True)
# zip dir_experiment_run
log.info(
f'zipping {flags.dir_experiment_run} '
f'to {(Path(dir_experiment) / flags.experiment_uid).with_suffix(".zip")}.'
)
dir_experiment_zipped = (dir_experiment / flags.experiment_uid)
shutil.make_archive(dir_experiment_zipped,
'zip',
flags.dir_experiment_run,
verbose=True)
assert dir_experiment_zipped.with_suffix('.zip').exists(), f'{dir_experiment_zipped} does not exist. ' \
f'Zipping of dir_experiment_run failed.'
# delete not compressed experiment dir
shutil.rmtree(str(flags.dir_experiment_run))
def load_experiment_from_db(_id: str, add_args=None):
"""Load experiment with flags and trained model from old experiment."""
dataset = _id.split('_')[0].lower()
flags_setup = BaseFlagsSetup(get_config_path(dataset=dataset))
flags = flags_setup.load_old_flags(_id=_id, add_args={'save_figure': False, **add_args})
exp = get_experiment(flags)
# load networks from database
exp.mm_vae = exp.experiments_database.load_networks_from_db(exp.mm_vae)
return exp
from mmvae_hub import log
from mmvae_hub.celeba.CelebaTrainer import CelebaTrainer
from mmvae_hub.celeba.experiment import CelebaExperiment
from mmvae_hub.celeba.flags import parser, CelebaFlagsSetup
from mmvae_hub.leomed_utils.boilerplate import compress_experiment_run_dir
from mmvae_hub.utils.setup.flags_utils import get_config_path
DATASET = 'celeba'
if __name__ == '__main__':
flags = parser.parse_args()
flags_setup = CelebaFlagsSetup(
get_config_path(dataset=DATASET, flags=flags))
flags = flags_setup.setup(flags, additional_args={'dataset': DATASET})
with maybe_norby(flags.norby,
f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
mst = CelebaExperiment(flags)
mst.set_optimizer()
trainer = CelebaTrainer(mst)
trainer.run_epochs()
log.info('Done.')
# move zipped experiment_dir_run in TMPDIR to experiment_dir
if flags.leomed:
compress_experiment_run_dir(flags)
return {_class: i for i, _class in enumerate(self.classes)}
def _check_exists_mnist(self):
return (Path(self.processed_folder) / self.training_file_mnist).exists() and (
Path(self.processed_folder) / self.test_file_mnist).exists()
def _check_exists_svhn(self):
return (os.path.exists(os.path.join(self.dir_svhn,
self.training_file_svhn)) and
os.path.exists(os.path.join(self.dir_svhn,
self.test_file_svhn)))
@staticmethod
def extract_gzip(gzip_path, remove_finished=False):
print('Extracting {}'.format(gzip_path))
with open(gzip_path.replace('.gz', ''), 'wb') as out_f, \
gzip.GzipFile(gzip_path) as zip_f:
out_f.write(zip_f.read())
if remove_finished:
os.unlink(gzip_path)
def extra_repr(self):
return "Split: {}".format("Train" if self.train is True else "Test")
if __name__ == '__main__':
config = json2dict(Path(get_config_path(dataset='mnistsvhntext')))
download_zip_from_url(
url='https://www.dropbox.com/sh/lx8669lyok9ois6/AADMhr3EluBXJyZnV1_lYntTa/data_mnistsvhntext.zip?dl=1',
dest_folder=Path(config['dir_data']).expanduser().parent, verbose=True)
if __name__ == "__main__":
# parser = argparse.ArgumentParser()
# parser.add_argument('--seed', type=int, default=42)
# parser.add_argument('--num-modalities', type=int, default=5)
# parser.add_argument('--savepath-train', type=str, required=True)
# parser.add_argument('--savepath-test', type=str, required=True)
# parser.add_argument('--backgroundimagepath', type=str, required=True)
# args = parser.parse_args() # use vars to convert args into a dict
# print("\nARGS:\n", args)
from dataclasses import dataclass
from pathlib import Path
config = json2dict(get_config_path())
@dataclass
class Args:
savepath_train: Path = Path(config['dir_data']) / 'train'
savepath_test: Path = Path(config['dir_data']) / 'test'
backgroundimagepath: Path = Path(__file__).parent / 'polymnist_background_images'
num_modalities: int = 5
args = Args()
# create dataset
PolymnistDataset.create_polymnist_dataset(args.savepath_train, args.backgroundimagepath, args.num_modalities,
train=True)
PolymnistDataset.create_polymnist_dataset(args.savepath_test, args.backgroundimagepath, args.num_modalities,
"dataloader_workers": [16],
"max_beta": [0.],
"beta_start_epoch": [0.],
"beta_warmup": [50],
# "num_gfm_flows": [3],
# "coupling_dim": [32],
"coupling_dim": [64],
"num_gfm_flows": [3],
"nbr_coupling_block_layers": [8],
"end_epoch": [150],
"calc_nll": [False],
"K": [1],
"eval_freq": [150],
}
if __name__ == '__main__':
for grid in [sp_mopoe_mimic]:
for sp in ParameterGrid(grid):
# for _ in [1]:
flags = parser.parse_args()
flags_setup = MimicFlagsSetup(
get_config_path(dataset='mimic', flags=flags))
flags = flags_setup.setup(flags, additional_args=sp)
with norby(f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
mst = MimicExperiment(flags)
mst.set_optimizer()
trainer = MimicTrainer(mst)
trainer.run_epochs()
for l, line in enumerate(lines):
width, height = font.getsize(line)
draw.text((0, (h / 2) - (num_lines / 2 - l) * height),
line, (0, 0, 0),
font=font)
y_text += height
if imgsize[0] == 3:
text_pil = transforms.ToTensor()(pil_img.resize(
(imgsize[1], imgsize[2]), Image.ANTIALIAS))
else:
text_pil = transforms.ToTensor()(pil_img.resize(
(imgsize[1], imgsize[2]), Image.ANTIALIAS).convert('L'))
return text_pil
if __name__ == '__main__':
config = json2dict(get_config_path(dataset='celeba'))
img_clf_path = Path(config['dir_clf']).expanduser() / 'clf_celeba_text.pth'
img_clf_path.parent.mkdir(exist_ok=True, parents=True)
if not img_clf_path.exists():
print(
f'text clf not found under {img_clf_path}. Parent folder contains: {list(img_clf_path.parent.iterdir())}'
)
download_from_url(
url=
'https://www.dropbox.com/sh/lx8669lyok9ois6/AACaBy1YNNq3ebh149k_EXrca/trained_classifiers/trained_clfs_celeba/clf_m2?dl=1',
dest_path=img_clf_path,
verbose=True)
# "factorized_representation": [True],
# "beta": [5.],
# "beta_style": [2.0],
# "beta_content": [1.0],
# "beta_m1_style": [1.0],
# "beta_m2_style": [5.0],
# "style_img_dim": [32],
# "style_text_dim": [32],
}
if __name__ == '__main__':
dataset = 'celeba'
for grid in [sp_joint_elbo_article]:
for sp in ParameterGrid(grid):
# for _ in [1]:
flags = parser.parse_args()
flags_setup = CelebaFlagsSetup(
get_config_path(dataset='celeba', flags=flags))
flags = flags_setup.setup(flags,
additional_args={
**sp, 'dataset': dataset
})
with norby(f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
mst = CelebaExperiment(flags)
mst.set_optimizer()
trainer = CelebaTrainer(mst)
trainer.run_epochs()
from norby.utils import maybe_norby
from mmvae_hub.leomed_utils.boilerplate import compress_experiment_run_dir
from mmvae_hub.mimic.MimicTrainer import MimicTrainer
from mmvae_hub.mimic.experiment import MimicExperiment
from mmvae_hub.mimic.flags import parser, MimicFlagsSetup
from mmvae_hub.utils.setup.flags_utils import get_config_path
DATASET = 'mimic'
if __name__ == '__main__':
flags = parser.parse_args()
flags.dataset = 'mimic'
flags_setup = MimicFlagsSetup(get_config_path(dataset=DATASET,
flags=flags))
flags = flags_setup.setup(flags, additional_args={'dataset': DATASET})
with maybe_norby(flags.norby,
f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
exp = MimicExperiment(flags)
exp.set_optimizer()
trainer = MimicTrainer(exp)
trainer.run_epochs()
# move zipped experiment_dir_run in TMPDIR to experiment_dir
if flags.leomed:
compress_experiment_run_dir(flags)
method = 'mopgfm'
flags = parser.parse_args()
study_name = f'hyperopt-{method}2'
# storage_sqlite = optuna.storages.RDBStorage("sqlite:///hyperopt.db", heartbeat_interval=1)
# study = optuna.create_study(direction="maximize", storage=storage_sqlite,
# study_name=f"distributed-hyperopt-{flags.method}")
postgresql_storage_address = "postgresql://klugh@ethsec-login-03:5433/distributed_hyperopt"
try:
study = optuna.load_study(study_name=study_name,
storage=postgresql_storage_address)
except:
study = optuna.create_study(direction="maximize",
storage=postgresql_storage_address,
study_name=study_name)
flags.dir_experiment = Path(flags.dir_experiment) / 'optuna'
flags_setup = FlagsSetup(get_config_path(dataset=dataset, flags=flags))
trainer = HyperoptTrainer(flags,
flags_setup,
dataset=dataset,
method=method)
study.optimize(trainer.hyperopt, n_trials=100, gc_after_trial=True)
print("Best trial:")
print(study.best_params)
with open('hyperopt_best_results.json', 'w') as jsonfile:
json.dump(study.best_params, jsonfile)
search_space1 = {
'method': 'mopoe',
"initial_learning_rate": 0.0005,
'class_dim': 640,
"min_beta": 0,
"max_beta": 2.0,
"beta_warmup": 0,
"num_gfm_flows": 3,
"num_mods": 3,
"end_epoch": 10,
"eval_freq": 10,
"data_multiplications": 1
}
if __name__ == '__main__':
for grid in [search_spaces_amortized]:
for sp in ParameterGrid(grid):
# for sp in [grid]:
# for _ in [1]:
flags = parser.parse_args()
flags_setup = mnistsvhntextFlagsSetup(
get_config_path(dataset='mnistsvhntext', flags=flags))
flags = flags_setup.setup(flags, additional_args=sp)
with norby(f'Starting Experiment {flags.experiment_uid}.',
f'Experiment {flags.experiment_uid} finished.'):
mst = MNISTSVHNText(flags)
mst.set_optimizer()
trainer = mnistsvhnTrainer(mst)
trainer.run_epochs()
# (experiment_dir / 'checkpoints').iterdir()) == 0:
# print(f'removing dir {experiment_dir}')
# shutil.rmtree(experiment_dir)
#
# elif (max(int(d.name) for d in (experiment_dir / 'checkpoints').iterdir() if d.name.startswith('0')) < 10):
# print(f'removing dir {experiment_dir}')
# shutil.rmtree(experiment_dir)
def clean_early_checkpoints(parent_folder: Path):
for experiment_dir in parent_folder.iterdir():
checkpoints_dir = parent_folder / experiment_dir / 'checkpoints/0*'
checkpoints = glob.glob(checkpoints_dir.__str__())
checkpoint_epochs = sorted(
[Path(checkpoint).stem for checkpoint in checkpoints])
for checkpoint in checkpoints:
if Path(checkpoint).stem != checkpoint_epochs[-1]:
shutil.rmtree(checkpoint)
if __name__ == '__main__':
clean_database()
clean_database_model_checkpoints()
config_path_polymnist = get_config_path(dataset='polymnist')
config_path_mimic = get_config_path(dataset='mimic')
for config_path in [config_path_polymnist, config_path_polymnist]:
with open(config_path, 'rt') as json_file:
config = json.load(json_file)
clean_exp_dirs(config)
"initial_learning_rate": [0.001],
"end_epoch": [1],
"eval_freq": [1],
"calc_nll": [False],
}
search_space_sylvester = {
'method': ['mopoe'],
'max_beta': [1.],
'class_dim': [64],
"num_mods": [1],
"num_flows": [5],
"initial_learning_rate": [0.0005],
"end_epoch": [1],
"eval_freq": [1],
"calc_nll": [False],
}
if __name__ == '__main__':
for grid in [search_spaces_amortized]:
for sp in ParameterGrid(grid):
# for _ in [1]:
flags = parser.parse_args()
flags_setup = FlagsSetup(get_config_path(dataset='polymnist', flags=flags))
flags = flags_setup.setup(flags, additional_args=sp)
with norby(f'Starting Experiment {flags.experiment_uid}.', f'Experiment {flags.experiment_uid} finished.'):
mst = PolymnistExperiment(flags)
mst.set_optimizer()
trainer = PolymnistTrainer(mst)
trainer.run_epochs()