def get_eval_config():
"""Returns the hyperparameter configs for different experiments."""
arch_enc = h.fixed("encoder.encoder_fn", "@conv_encoder", length=1)
arch_dec = h.fixed("decoder.decoder_fn", "@deconv_decoder", length=1)
architecture = h.zipit([arch_enc, arch_dec])
return h.product([
get_datasets(),
])
def get_config():
"""Returns the hyperparameter configs for different experiments."""
arch_enc = h.fixed("encoder.encoder_fn", "@conv_encoder", length=1)
arch_dec = h.fixed("decoder.decoder_fn", "@deconv_decoder", length=1)
architecture = h.zipit([arch_enc, arch_dec])
return h.product([
get_datasets(), # 7
architecture, # 1
get_default_models(), # 6 models x 6 hyperparameters
get_seeds(50),
])
def get_default_models():
"""Our default set of models (6 model * 6 hyperparameters=36 models)."""
# BetaVAE config.
model_name = h.fixed("model.name", "beta_vae")
model_fn = h.fixed("model.model", "@vae()")
betas = h.sweep("vae.beta", h.discrete([0., 0.5, 1., 2., 4., 8.]))
params_product = h.product([betas])
config_vae = h.zipit([model_name, params_product, model_fn])
all_models = h.chainit([config_vae])
return all_models
def get_default_models():
"""Our default set of models (6 model * 6 hyperparameters=36 models)."""
# BetaVAE config.
model_name = h.fixed("model.name", "supervised_vae")
model_fn = h.fixed("model.model", "@supervised_vae()")
betas = h.sweep("supervised_vae.beta", h.discrete([0., 1., 4.]))
omegas = h.sweep("supervised_vae.omega", h.discrete([0.0, 1.0, 10.0]))
params_product = h.product([betas, omegas])
config_supervised_vae = h.zipit([model_name, params_product, model_fn])
all_models = h.chainit([config_supervised_vae])
return all_models
def get_default_models():
"""Our default set of models (6 model * 6 hyperparameters=36 models)."""
# BetaVAE config.
model_name = h.fixed("model.name", "balanced_beta_vae")
model_fn = h.fixed("model.model", "@balanced_beta_vae()")
beta_sizes = h.sweep("balanced_beta_vae.beta_size",
h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.]))
beta_variances = h.sweep("balanced_beta_vae.beta_variance",
h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.]))
betas_product = h.product([beta_sizes, beta_variances])
config_balanced_beta_vae = h.zipit([model_name, betas_product, model_fn])
all_models = h.chainit([config_balanced_beta_vae])
return all_models
def get_default_models():
"""Our default set of models (6 model * 6 hyperparameters=36 models)."""
# BetaVAE config.
model_name = h.fixed("model.name", "augmented_variance_vae")
model_fn = h.fixed("model.model", "@augmented_variance_vae()")
mean_var_weights = h.sweep("augmented_variance_vae.mean_var_weight", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.]))
variance_weights = h.sweep("augmented_variance_vae.variance_weight", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.]))
weights_product = h.product([mean_var_weights, variance_weights])
config_balanced_beta_vae = h.zipit([model_name, weights_product, model_fn])
all_models = h.chainit([
config_balanced_beta_vae
])
return all_models
import disentanglement_lib.utils.hyperparams as h
_betas = h.sweep('beta', (1, 2, 4, 6, 8, 16, 32, 64))
_datasets = h.sweep('dataset', (
"dsprites_full",
"color_dsprites",
"noisy_dsprites",
"scream_dsprites",
"smallnorb",
"cars3d",
"shapes3d",
))
_all_layers = h.sweep('all_layers', (True, False))
_scale_per_layer = h.sweep('scale_per_layer', (True, False))
_dims = h.sweep('dim', ('col', 'row'))
_sweep_dim_wise = h.product((_betas, _datasets, _all_layers, _scale_per_layer, _dims))
_dim_wise_studies = {
f"{s['dataset']}_dim_wise_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseL1SparsityStudy(
**s)
for s in _sweep_dim_wise
}
_dim_wise_mask_studies = {
f"{s['dataset']}_dim_wise_mask_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseMaskL1Study(
**s)
for s in _sweep_dim_wise
}
_dim_wise_mask_studies_2 = {
f"{s['dataset']}_dim_wise_mask_2_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseMaskL1Study(
lmbd_l1_range=np.logspace(-7, -5, 3, endpoint=False),
**s)
def main():
sweep = h.product((
h.sweep('dataset', DATASETS),
h.sweep('method', METHODS),
h.sweep(
'beta',
(
# 1,
# 4,
# 8,
16,
# 32,
)),
h.sweep(
'all_layers',
(
True,
# False,
)),
h.sweep(
'scale',
(
True,
# False,
)),
h.sweep('anneal', (
True,
False,
)),
))
df_list = []
model_names = set()
for setting in sweep:
dataset, method, beta, all_layers, scale, anneal = setting['dataset'], setting['method'], setting['beta'], \
setting[
'all_layers'], setting['scale'], setting['anneal']
if 'masked' in method or 'small' in method or 'weight_decay' in method or 'proximal' in method or 'wae' in method:
scale = False
out_dir = PLOT_DIR / (method + ('_anneal' if anneal else '') +
('_all' if all_layers else '') +
('_scale' if scale else '')) / f'beta_{beta}'
df = load_results()
if 'dim_wise_mask_l1' in method:
dim_wise_df = df.loc[df[MODEL_COL_STR].str.contains(
'dim_wise_mask_l1')]
dim_wise_df[MODEL_COL_STR] = dim_wise_df.apply(
lambda row: row[MODEL_COL_STR].replace(
'l1', 'l1_' + row['train_config.dim_wise_l1_vae.dim'].
replace("'", '')),
axis=1,
)
df.loc[df[MODEL_COL_STR].str.contains(
'dim_wise_mask_l1')] = dim_wise_df
df = df.loc[df[MODEL_COL_STR].str.contains(method)]
if 'dim_wise_mask_l1' in method:
if not ('row' in method or 'col' in method):
# non dim wise l1
df = df.loc[df['train_config.dim_wise_l1_vae.dim'] == 'None']
idxs_all_layers = (
df['train_config.dim_wise_l1_vae.all_layers']
== 'True') | (df['train_config.conv_encoder.all_layers'] == 'True')
df = df.loc[idxs_all_layers] if all_layers else df.loc[~idxs_all_layers]
idxs_scale = (df[get_scale_col_name(method)] == 'True')
df = df.loc[idxs_scale] if scale else df.loc[~idxs_scale]
idxs_anneal = (~df['train_config.vd_vae.anneal_kld_for'].isin(
('None', None)))
df = df.loc[idxs_anneal] if anneal else df.loc[~idxs_anneal]
print(dataset, out_dir.parts[1:])
print(len(df))
if len(df) < 1:
continue
if 'dim_wise' in method:
# TODO nicer way of mapping reg. strengths
df[MODEL_COL_STR] = df[MODEL_COL_STR] + '_' + df[
'train_config.dim_wise_l1_vae.lmbd_l1'].map("{:.2e}".format)
df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace(
'dim_wise_mask_l1_', '')
df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace('_vae', '')
# df = df.loc[df['train_config.dim_wise_l1_vae.lmbd_l1'].between(1e-10, 1e0)]
df = df.loc[df['train_config.dim_wise_l1_vae.lmbd_l1'].isin((
# 1e-10,
# 3.1622776601683795e-10,
# 1e-09,
# 3.1622776601683795e-09,
# 1e-08,
# 3.162277660168379e-08,
# 1e-07,
# 4.641588833612782e-07,
1e-06,
# 3.162277660168379e-06,
1e-05,
# 3.727593720314938e-05,
0.0001,
# 0.00031622776601683794,
0.001,
# 0.0031622776601683794,
0.01,
# 0.03162277660168379,
0.1,
# 0.31622776601683794,
# 1.0,
))]
reg_weight_col = 'train_config.dim_wise_l1_vae.lmbd_l1'
elif method == 'masked':
df[MODEL_COL_STR] = df[MODEL_COL_STR] + '_' + df[
'train_config.conv_encoder.perc_sparse'].map(
lambda x: round(float(x), 2)).map(str)
reg_weight_col = 'train_config.conv_encoder.perc_sparse'
df[reg_weight_col] = pd.to_numeric(df[reg_weight_col])
# TODO use get_reg_col_name() fn here
elif 'vd' in method:
reg_weight_col = 'train_config.vd_vae.lmbd_kld_vd'
elif 'proximal' in method:
reg_weight_col = 'train_config.proximal_vae.lmbd_prox'
elif 'softmax' in method:
reg_weight_col = 'train_config.conv_encoder.softmax_temperature'
elif 'wae' in method:
reg_weight_col = 'train_config.vae.beta'
new_reg_weight_col = reg_weight_col.replace('vae', 'wae')
df[new_reg_weight_col] = df[reg_weight_col]
df[reg_weight_col] = beta
reg_weight_col = new_reg_weight_col
# ablation test methods
elif 'weight_decay' in method:
reg_weight_col = 'train_config.dim_wise_l1_vae.lmbd_l2'
elif 'small_vae' in method:
reg_weight_col = 'train_config.conv_encoder.perc_units'
if dataset == 'shapes3d':
dlib_df = load_results()
dlib_df = dlib_df.loc[dlib_df[MODEL_COL_STR] == "'beta_vae'"]
else:
dlib_df = load_dlib_df()
model_name = out_dir.parent.name
df[MODEL_COL_STR] = model_name
model_names.add(model_name)
dlib_df[reg_weight_col] = 0
df = pd.concat((df, dlib_df), sort=True)
df = df.loc[(df['train_config.vae.beta'] == beta)
& (df[DATASET_COL_STR] == f"'{dataset}'")]
if len(df[MODEL_COL_STR].unique()) < 2:
continue
df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace("'", '')
# out_dir.mkdir(exist_ok=True, parents=True)
# plot_results(
# df=df,
# dataset=dataset,
# out_dir=out_dir,
# reg_weight_col=reg_weight_col,
# )
df = df.loc[df[reg_weight_col] != 0]
df_list.append(df)
df = pd.concat(df_list)
shapes_baseline = load_results()
shapes_baseline = shapes_baseline.loc[
(shapes_baseline[DATASET_COL_STR].str.contains('shapes3d'))
& (shapes_baseline[MODEL_COL_STR].str.contains('beta_vae'))]
df = pd.concat((df, load_dlib_df(), shapes_baseline))
df = df.loc[df['train_config.vae.beta'] == 16]
top_k_groups = None
# plot_fig_15(df, methods=model_names, top_k_groups=top_k_groups)
# plot_fig_16(df, methods=METHODS[:3] + ('beta_vae',))
# plot_fig_16(df, methods=[*model_names, 'beta_vae'], top_k_groups=top_k_groups)
# plot_fig_16(df, methods=model_names, top_k_groups=top_k_groups, diff_over_baseline=True)
# plot_fig_17(df, methods=METHODS[:3] + ('beta_vae',))
# plot_fig_17(df, methods=[*model_names, 'beta_vae'], top_k_groups=top_k_groups)
plot_fig_17(df,
methods=model_names,
top_k_groups=top_k_groups,
diff_over_baseline=True)
