def get_eval_config(): """Returns the hyperparameter configs for different experiments.""" arch_enc = h.fixed("encoder.encoder_fn", "@conv_encoder", length=1) arch_dec = h.fixed("decoder.decoder_fn", "@deconv_decoder", length=1) architecture = h.zipit([arch_enc, arch_dec]) return h.product([ get_datasets(), ])
def get_config(): """Returns the hyperparameter configs for different experiments.""" arch_enc = h.fixed("encoder.encoder_fn", "@conv_encoder", length=1) arch_dec = h.fixed("decoder.decoder_fn", "@deconv_decoder", length=1) architecture = h.zipit([arch_enc, arch_dec]) return h.product([ get_datasets(), # 7 architecture, # 1 get_default_models(), # 6 models x 6 hyperparameters get_seeds(50), ])
def get_default_models(): """Our default set of models (6 model * 6 hyperparameters=36 models).""" # BetaVAE config. model_name = h.fixed("model.name", "beta_vae") model_fn = h.fixed("model.model", "@vae()") betas = h.sweep("vae.beta", h.discrete([0., 0.5, 1., 2., 4., 8.])) params_product = h.product([betas]) config_vae = h.zipit([model_name, params_product, model_fn]) all_models = h.chainit([config_vae]) return all_models
def get_default_models(): """Our default set of models (6 model * 6 hyperparameters=36 models).""" # BetaVAE config. model_name = h.fixed("model.name", "supervised_vae") model_fn = h.fixed("model.model", "@supervised_vae()") betas = h.sweep("supervised_vae.beta", h.discrete([0., 1., 4.])) omegas = h.sweep("supervised_vae.omega", h.discrete([0.0, 1.0, 10.0])) params_product = h.product([betas, omegas]) config_supervised_vae = h.zipit([model_name, params_product, model_fn]) all_models = h.chainit([config_supervised_vae]) return all_models
def get_default_models(): """Our default set of models (6 model * 6 hyperparameters=36 models).""" # BetaVAE config. model_name = h.fixed("model.name", "balanced_beta_vae") model_fn = h.fixed("model.model", "@balanced_beta_vae()") beta_sizes = h.sweep("balanced_beta_vae.beta_size", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.])) beta_variances = h.sweep("balanced_beta_vae.beta_variance", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.])) betas_product = h.product([beta_sizes, beta_variances]) config_balanced_beta_vae = h.zipit([model_name, betas_product, model_fn]) all_models = h.chainit([config_balanced_beta_vae]) return all_models
def get_default_models(): """Our default set of models (6 model * 6 hyperparameters=36 models).""" # BetaVAE config. model_name = h.fixed("model.name", "augmented_variance_vae") model_fn = h.fixed("model.model", "@augmented_variance_vae()") mean_var_weights = h.sweep("augmented_variance_vae.mean_var_weight", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.])) variance_weights = h.sweep("augmented_variance_vae.variance_weight", h.discrete([0.0, 0.1, 0.5, 1., 2., 4., 8., 16.])) weights_product = h.product([mean_var_weights, variance_weights]) config_balanced_beta_vae = h.zipit([model_name, weights_product, model_fn]) all_models = h.chainit([ config_balanced_beta_vae ]) return all_models
import disentanglement_lib.utils.hyperparams as h _betas = h.sweep('beta', (1, 2, 4, 6, 8, 16, 32, 64)) _datasets = h.sweep('dataset', ( "dsprites_full", "color_dsprites", "noisy_dsprites", "scream_dsprites", "smallnorb", "cars3d", "shapes3d", )) _all_layers = h.sweep('all_layers', (True, False)) _scale_per_layer = h.sweep('scale_per_layer', (True, False)) _dims = h.sweep('dim', ('col', 'row')) _sweep_dim_wise = h.product((_betas, _datasets, _all_layers, _scale_per_layer, _dims)) _dim_wise_studies = { f"{s['dataset']}_dim_wise_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseL1SparsityStudy( **s) for s in _sweep_dim_wise } _dim_wise_mask_studies = { f"{s['dataset']}_dim_wise_mask_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseMaskL1Study( **s) for s in _sweep_dim_wise } _dim_wise_mask_studies_2 = { f"{s['dataset']}_dim_wise_mask_2_l1_{s['dim']}_{'all_' if s['all_layers'] else ''}{'scale_' if s['scale_per_layer'] else ''}b_{s['beta']}": sparsity_study.DimWiseMaskL1Study( lmbd_l1_range=np.logspace(-7, -5, 3, endpoint=False), **s)
def main(): sweep = h.product(( h.sweep('dataset', DATASETS), h.sweep('method', METHODS), h.sweep( 'beta', ( # 1, # 4, # 8, 16, # 32, )), h.sweep( 'all_layers', ( True, # False, )), h.sweep( 'scale', ( True, # False, )), h.sweep('anneal', ( True, False, )), )) df_list = [] model_names = set() for setting in sweep: dataset, method, beta, all_layers, scale, anneal = setting['dataset'], setting['method'], setting['beta'], \ setting[ 'all_layers'], setting['scale'], setting['anneal'] if 'masked' in method or 'small' in method or 'weight_decay' in method or 'proximal' in method or 'wae' in method: scale = False out_dir = PLOT_DIR / (method + ('_anneal' if anneal else '') + ('_all' if all_layers else '') + ('_scale' if scale else '')) / f'beta_{beta}' df = load_results() if 'dim_wise_mask_l1' in method: dim_wise_df = df.loc[df[MODEL_COL_STR].str.contains( 'dim_wise_mask_l1')] dim_wise_df[MODEL_COL_STR] = dim_wise_df.apply( lambda row: row[MODEL_COL_STR].replace( 'l1', 'l1_' + row['train_config.dim_wise_l1_vae.dim']. replace("'", '')), axis=1, ) df.loc[df[MODEL_COL_STR].str.contains( 'dim_wise_mask_l1')] = dim_wise_df df = df.loc[df[MODEL_COL_STR].str.contains(method)] if 'dim_wise_mask_l1' in method: if not ('row' in method or 'col' in method): # non dim wise l1 df = df.loc[df['train_config.dim_wise_l1_vae.dim'] == 'None'] idxs_all_layers = ( df['train_config.dim_wise_l1_vae.all_layers'] == 'True') | (df['train_config.conv_encoder.all_layers'] == 'True') df = df.loc[idxs_all_layers] if all_layers else df.loc[~idxs_all_layers] idxs_scale = (df[get_scale_col_name(method)] == 'True') df = df.loc[idxs_scale] if scale else df.loc[~idxs_scale] idxs_anneal = (~df['train_config.vd_vae.anneal_kld_for'].isin( ('None', None))) df = df.loc[idxs_anneal] if anneal else df.loc[~idxs_anneal] print(dataset, out_dir.parts[1:]) print(len(df)) if len(df) < 1: continue if 'dim_wise' in method: # TODO nicer way of mapping reg. strengths df[MODEL_COL_STR] = df[MODEL_COL_STR] + '_' + df[ 'train_config.dim_wise_l1_vae.lmbd_l1'].map("{:.2e}".format) df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace( 'dim_wise_mask_l1_', '') df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace('_vae', '') # df = df.loc[df['train_config.dim_wise_l1_vae.lmbd_l1'].between(1e-10, 1e0)] df = df.loc[df['train_config.dim_wise_l1_vae.lmbd_l1'].isin(( # 1e-10, # 3.1622776601683795e-10, # 1e-09, # 3.1622776601683795e-09, # 1e-08, # 3.162277660168379e-08, # 1e-07, # 4.641588833612782e-07, 1e-06, # 3.162277660168379e-06, 1e-05, # 3.727593720314938e-05, 0.0001, # 0.00031622776601683794, 0.001, # 0.0031622776601683794, 0.01, # 0.03162277660168379, 0.1, # 0.31622776601683794, # 1.0, ))] reg_weight_col = 'train_config.dim_wise_l1_vae.lmbd_l1' elif method == 'masked': df[MODEL_COL_STR] = df[MODEL_COL_STR] + '_' + df[ 'train_config.conv_encoder.perc_sparse'].map( lambda x: round(float(x), 2)).map(str) reg_weight_col = 'train_config.conv_encoder.perc_sparse' df[reg_weight_col] = pd.to_numeric(df[reg_weight_col]) # TODO use get_reg_col_name() fn here elif 'vd' in method: reg_weight_col = 'train_config.vd_vae.lmbd_kld_vd' elif 'proximal' in method: reg_weight_col = 'train_config.proximal_vae.lmbd_prox' elif 'softmax' in method: reg_weight_col = 'train_config.conv_encoder.softmax_temperature' elif 'wae' in method: reg_weight_col = 'train_config.vae.beta' new_reg_weight_col = reg_weight_col.replace('vae', 'wae') df[new_reg_weight_col] = df[reg_weight_col] df[reg_weight_col] = beta reg_weight_col = new_reg_weight_col # ablation test methods elif 'weight_decay' in method: reg_weight_col = 'train_config.dim_wise_l1_vae.lmbd_l2' elif 'small_vae' in method: reg_weight_col = 'train_config.conv_encoder.perc_units' if dataset == 'shapes3d': dlib_df = load_results() dlib_df = dlib_df.loc[dlib_df[MODEL_COL_STR] == "'beta_vae'"] else: dlib_df = load_dlib_df() model_name = out_dir.parent.name df[MODEL_COL_STR] = model_name model_names.add(model_name) dlib_df[reg_weight_col] = 0 df = pd.concat((df, dlib_df), sort=True) df = df.loc[(df['train_config.vae.beta'] == beta) & (df[DATASET_COL_STR] == f"'{dataset}'")] if len(df[MODEL_COL_STR].unique()) < 2: continue df[MODEL_COL_STR] = df[MODEL_COL_STR].str.replace("'", '') # out_dir.mkdir(exist_ok=True, parents=True) # plot_results( # df=df, # dataset=dataset, # out_dir=out_dir, # reg_weight_col=reg_weight_col, # ) df = df.loc[df[reg_weight_col] != 0] df_list.append(df) df = pd.concat(df_list) shapes_baseline = load_results() shapes_baseline = shapes_baseline.loc[ (shapes_baseline[DATASET_COL_STR].str.contains('shapes3d')) & (shapes_baseline[MODEL_COL_STR].str.contains('beta_vae'))] df = pd.concat((df, load_dlib_df(), shapes_baseline)) df = df.loc[df['train_config.vae.beta'] == 16] top_k_groups = None # plot_fig_15(df, methods=model_names, top_k_groups=top_k_groups) # plot_fig_16(df, methods=METHODS[:3] + ('beta_vae',)) # plot_fig_16(df, methods=[*model_names, 'beta_vae'], top_k_groups=top_k_groups) # plot_fig_16(df, methods=model_names, top_k_groups=top_k_groups, diff_over_baseline=True) # plot_fig_17(df, methods=METHODS[:3] + ('beta_vae',)) # plot_fig_17(df, methods=[*model_names, 'beta_vae'], top_k_groups=top_k_groups) plot_fig_17(df, methods=model_names, top_k_groups=top_k_groups, diff_over_baseline=True)