def main(argv):
del argv # Unused
# Save all results in subdirectories of following path
base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), FLAGS.base_dir)
# Overwrite output or not (for rerunning script)
overwrite = True
# Results directory of BetaTCVAE
path_adagvae = os.path.join(base_path,FLAGS.output_dir)
gin_bindings = [
"model.random_seed = {}".format(FLAGS.seed),
"subset.name = '{}'".format(FLAGS.subset),
"encoder.num_latent = {}".format(FLAGS.dim)
]
# Train model. Training is configured with a gin config
train.train_with_gin(os.path.join(path_adagvae, 'model'), overwrite,
['baselines/adagvae/adagvae_train.gin'], gin_bindings)
# Extract mean representation of latent space
representation_path = os.path.join(path_adagvae, "representation")
model_path = os.path.join(path_adagvae, "model")
postprocess_gin = ['baselines/adagvae/adagvae_postprocess.gin'] # This contains the settings.
postprocess.postprocess_with_gin(model_path, representation_path, overwrite,
postprocess_gin)
# Compute DCI metric
result_path = os.path.join(path_adagvae, "metrics", "dci")
representation_path = os.path.join(path_adagvae, "representation")
evaluate.evaluate_with_gin(representation_path, result_path, overwrite, ['baselines/adagvae/adagvae_dci.gin'])
def setUp(self):
super(PostprocessTest, self).setUp()
self.model_dir = self.create_tempdir(
"model", cleanup=absltest.TempFileCleanup.OFF).full_path
train.train_with_gin(self.model_dir, True, [
resources.get_file(
"config/tests/methods/unsupervised/train_test.gin")
], [])
def setUp(self):
super(EvaluateTest, self).setUp()
self.model_dir = self.create_tempdir(
"model", cleanup=absltest.TempFileCleanup.OFF).full_path
model_config = resources.get_file(
"config/tests/methods/unsupervised/train_test.gin")
train.train_with_gin(self.model_dir, True, [model_config])
self.output_dir = self.create_tempdir(
"output", cleanup=absltest.TempFileCleanup.OFF).full_path
postprocess_config = resources.get_file(
"config/tests/postprocessing/postprocess_test_configs/mean.gin")
postprocess.postprocess_with_gin(self.model_dir, self.output_dir, True,
[postprocess_config])
def setUp(self):
super(EvaluateTest, self).setUp()
self.model1_dir = self.create_tempdir(
"model1/model", cleanup=absltest.TempFileCleanup.OFF).full_path
self.model2_dir = self.create_tempdir(
"model2/model", cleanup=absltest.TempFileCleanup.OFF).full_path
model_config = resources.get_file(
"config/tests/methods/unsupervised/train_test.gin")
gin.clear_config()
train.train_with_gin(self.model1_dir, True, [model_config])
train.train_with_gin(self.model2_dir, True, [model_config])
self.output_dir = self.create_tempdir(
"output", cleanup=absltest.TempFileCleanup.OFF).full_path
def test_visualize_sigmoid(self, activation):
activation_binding = (
"reconstruction_loss.activation = '{}'".format(activation))
self.model_dir = self.create_tempdir(
"model_{}".format(activation),
cleanup=absltest.TempFileCleanup.OFF).full_path
train.train_with_gin(self.model_dir, True, [
resources.get_file("config/tests/methods/unsupervised/train_test.gin")
], [activation_binding])
visualize_model.visualize(
self.model_dir,
self.create_tempdir("visualization_{}".format(activation)).full_path,
True,
num_animations=1,
num_frames=4)
def train_tf_main():
aicrowd_helpers.register_progress(0.0)
start_time = time.time()
train.train_with_gin(os.path.join(experiment_output_path, "model"),
overwrite, [get_full_path("model.gin")], gin_bindings)
# path=os.path.join(experiment_output_path, str(time.time()))
# train.train_with_gin(
# path, overwrite,
# [get_full_path("model.gin")], gin_bindings)
elapsed_time = time.time() - start_time
print(
"##################################Elapsed TIME##############################"
)
print(elapsed_time)
print(
"##################################Elapsed TIME##############################"
)
########################################################################
# Register Progress (end of training, start of representation extraction)
########################################################################
aicrowd_helpers.register_progress(0.90)
# Extract the mean representation for both of these models.
representation_path = os.path.join(experiment_output_path,
"representation")
model_path = os.path.join(experiment_output_path, "model")
# model_path =path
# representation_path=path
# This contains the settings:
postprocess_gin = [get_full_path("postprocess.gin")]
postprocess.postprocess_with_gin(model_path, representation_path,
overwrite, postprocess_gin)
print("Written output to : ", experiment_output_path)
########################################################################
# Register Progress (of representation extraction)
########################################################################
aicrowd_helpers.register_progress(1.0)
########################################################################
# Submit Results for evaluation
########################################################################
cuda.close()
aicrowd_helpers.submit()
return elapsed_time, gin_bindings
# By default, we do not overwrite output directories. Set this to True, if you
# want to overwrite (in particular, if you rerun this script several times).
overwrite = False
# 1. Train a standard VAE (already implemented in disentanglement_lib).
# ------------------------------------------------------------------------------
# We save the results in a `vae` subfolder.
path_vae = os.path.join(base_path, "vae")
# The main training protocol of disentanglement_lib is defined in the
# disentanglement_lib.methods.unsupervised.train module. To configure
# training we need to provide a gin config. For a standard VAE, you may have a
# look at model.gin on how to do this.
train.train_with_gin(os.path.join(path_vae, "model"), overwrite, ["model.gin"])
# After this command, you should have a `vae` subfolder with a model that was
# trained for a few steps (in reality, you will want to train many more steps).
# 2. Train a custom VAE model.
# ------------------------------------------------------------------------------
# To train a custom model, we have to provide an implementation of the class
# GaussianEncoderModel in the
# disentanglement_lib.methods.unsupervised.gaussian_encoder_model module.
# For simplicty, we will subclass the BaseVAE class in
# disentanglement_lib.methods.unsupervised.vae which will train a VAE style
# model where the loss is given by a reconstruction loss (configured via gin)
# plus a custom regularizer (needs to be implemented.)
@gin.configurable("BottleneckVAE"
) # This will allow us to reference the model.
# # Two versions of the model are exported:
# # - one for "test" mode (the default tag)
# # - one for "training" mode ("is_training" tag)
# # In the case that the encoder/decoder have dropout, or BN layers, these two
# # graphs are different.
# tags_and_args = [
# ({"train"}, {"is_training": True}),
# (set(), {"is_training": False}),
# ]
# spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args,
# drop_collections=drop_collections)
# spec.export(export_path, checkpoint_path=checkpoint_path)
########################################################################
aicrowd_helpers.register_progress(0.0)
start_time = time.time()
train.train_with_gin(os.path.join(experiment_output_path, "model"), overwrite,
[get_full_path("model.gin")], gin_bindings)
# path=os.path.join(experiment_output_path, str(time.time()))
# train.train_with_gin(
# path, overwrite,
# [get_full_path("model.gin")], gin_bindings)
elapsed_time = time.time() - start_time
print(
"##################################Elapsed TIME##############################"
)
print(elapsed_time)
print(
"##################################Elapsed TIME##############################"
)
########################################################################
# Register Progress (end of training, start of representation extraction)
########################################################################
# By default, we do not overwrite output directories. Set this to True, if you
# want to overwrite (in particular, if you rerun this script several times).
overwrite = True
# 1. Train a standard VAE (already implemented in disentanglement_lib).
# ------------------------------------------------------------------------------
# We save the results in a `vae` subfolder.
dataset = "modelnet"
repetitions = 1
model_name_list = ["tcvae", "betavae", "factorvae"]
print("Running for {} repetitions".format(repetitions))
for model_name in model_name_list:
print("Running {}".format(model_name))
for repetition in range(repetitions):
print("Training repetition {}".format(repetition))
path_vae = os.path.join(base_path, dataset +"_" + model_name + "_" + str(repetition))
model_path = os.path.join(path_vae, model_name)
print("Model path {}".format(model_path))
print("Dataset {}".format(dataset))
print("Model name {}".format(model_name))
print([dataset+"_" + model_name + ".gin"])
train.train_with_gin(os.path.join(model_path, "model_name"), overwrite, ["model.gin"])
train.train_with_gin(os.path.join(model_path, "model_name"), overwrite, [dataset+"_" + model_name + ".gin"])
def test_train_model(self, gin_configs, gin_bindings):
# We clear the gin config before running. Otherwise, if a prior test fails,
# the gin config is locked and the current test fails.
gin.clear_config()
train.train_with_gin(self.create_tempdir().full_path, True, gin_configs,
gin_bindings)
def main(unused_argv):
# Obtain the study to reproduce.
study = reproduce.STUDIES[FLAGS.study]
dataset_names = ["cars3d", "smallnorb"]
for dataset_name in dataset_names:
postprocess_config_files = sorted(study.get_postprocess_config_files())
for beta in [1e-3, 1e-2, 0.1, 1, 10, 100, 1000]:
# Set correct output directory.
if FLAGS.output_directory is None:
output_directory = os.path.join("output", "{study}",
dataset_name, "{beta}")
else:
output_directory = FLAGS.output_directory
# Insert model number and study name into path if necessary.
output_directory = output_directory.format(
beta=str(beta), study="test_benchmark-experiment-6.1")
# Model training (if model directory is not provided).
model_bindings, model_config_file = get_model_configs(
beta, dataset_name)
logging.info("Training model...")
model_dir = os.path.join(output_directory, "model")
model_bindings = [
"model.name = '{}'".format(
os.path.basename(model_config_file)).replace(".gin",
""), # ,
# "model.model_num = {}".format(FLAGS.model_num),
] + model_bindings
train.train_with_gin(model_dir, FLAGS.overwrite,
[model_config_file], model_bindings)
# We visualize reconstructions, samples and latent space traversals.
visualize_dir = os.path.join(output_directory, "visualizations")
visualize_model.visualize(model_dir, visualize_dir,
FLAGS.overwrite)
# We extract the different representations and save them to disk.
random_state = np.random.RandomState(0)
postprocess_config_files = sorted(
study.get_postprocess_config_files())
for config in postprocess_config_files:
post_name = os.path.basename(config).replace(".gin", "")
logging.info("Extracting representation %s...", post_name)
post_dir = os.path.join(output_directory, "postprocessed",
post_name)
postprocess_bindings = [
"postprocess.random_seed = {}".format(
random_state.randint(2**16)),
"postprocess.name = '{}'".format(post_name)
]
postprocess.postprocess_with_gin(model_dir, post_dir,
FLAGS.overwrite, [config],
postprocess_bindings)
#Get representations and save to disk
gin.parse_config_files_and_bindings(
[], ["dataset.name = {}".format("'{}'".format(dataset_name))])
dataset = named_data.get_named_ground_truth_data()
factors, reps = get_representations(dataset, post_dir,
dataset_name)
pickle.dump(factors, open(os.path.join(post_dir, "factors.p"),
"wb"))
pickle.dump(reps, open(os.path.join(post_dir, "reps.p"), "wb"))
gin.clear_config()
def main():
parser = argparse.ArgumentParser(description='Project description.')
parser.add_argument('--result_dir',
help='Results directory.',
type=str,
default='/mnt/hdd/repo_results/Ramiel/sweep')
parser.add_argument('--study',
help='Name of the study.',
type=str,
default='unsupervised_study_v1')
parser.add_argument('--model_gin',
help='Name of the gin config.',
type=str,
default='test_model.gin')
parser.add_argument('--model_name',
help='Name of the model.',
type=str,
default='GroupVAE')
parser.add_argument('--vae_beta',
help='Beta-VAE beta.',
type=str,
default='1')
parser.add_argument('--hyps',
help='Hyperparameters of rec_mat_oth_spl_seed.',
type=str,
default='1_1_1_1_1_0')
parser.add_argument('--overwrite',
help='Whether to overwrite output directory.',
type=_str_to_bool,
default=False)
parser.add_argument('--dataset',
help='Dataset.',
type=str,
default='dsprites_full')
parser.add_argument('--recons_type',
help='Reconstruction loss type.',
type=str,
default='bernoulli_loss')
args = parser.parse_args()
# 1. Settings
study = reproduce.STUDIES[args.study]
args.hyps = args.hyps.split('_')
print()
study.print_postprocess_config()
print()
study.print_eval_config()
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus),
"Logical GPUs")
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
# Call training module to train the custom model.
if args.model_name == "GroupVAE":
dir_name = "GroupVAE-" + "-".join(args.hyps)
elif args.model_name == "vae":
dir_name = "LieVAE-" + args.vae_beta + "-" + args.hyps[5]
output_directory = os.path.join(args.result_dir, dir_name)
model_dir = os.path.join(output_directory, "model")
gin_bindings = [
"model.model = @" + args.model_name + "()",
"vae.beta = " + args.vae_beta, "GroupVAE.hy_rec = " + args.hyps[0],
"GroupVAE.hy_mat = " + args.hyps[1],
"GroupVAE.hy_oth = " + args.hyps[2],
"GroupVAE.hy_spl = " + args.hyps[3],
"GroupVAE.hy_ncut = " + args.hyps[4],
"model.random_seed = " + args.hyps[5],
"dataset.name = '" + args.dataset + "'",
"reconstruction_loss.loss_fn = @" + args.recons_type
]
train.train_with_gin(model_dir, args.overwrite, [args.model_gin],
gin_bindings)
# We fix the random seed for the postprocessing and evaluation steps (each
# config gets a different but reproducible seed derived from a master seed of
# 0). The model seed was set via the gin bindings and configs of the study.
random_state = np.random.RandomState(0)
# We extract the different representations and save them to disk.
postprocess_config_files = sorted(study.get_postprocess_config_files())
for config in postprocess_config_files:
post_name = os.path.basename(config).replace(".gin", "")
print("Extracting representation " + post_name + "...")
post_dir = os.path.join(output_directory, "postprocessed", post_name)
postprocess_bindings = [
"postprocess.random_seed = {}".format(random_state.randint(2**32)),
"postprocess.name = '{}'".format(post_name)
]
postprocess.postprocess_with_gin(model_dir, post_dir, args.overwrite,
[config], postprocess_bindings)
# Iterate through the disentanglement metrics.
eval_configs = sorted(study.get_eval_config_files())
blacklist = ['downstream_task_logistic_regression.gin']
# blacklist = [
# 'downstream_task_logistic_regression.gin', 'beta_vae_sklearn.gin',
# 'dci.gin', 'downstream_task_boosted_trees.gin', 'mig.gin',
# 'modularity_explicitness.gin', 'sap_score.gin', 'unsupervised.gin'
# ]
for config in postprocess_config_files:
post_name = os.path.basename(config).replace(".gin", "")
post_dir = os.path.join(output_directory, "postprocessed", post_name)
# Now, we compute all the specified scores.
for gin_eval_config in eval_configs:
if os.path.basename(gin_eval_config) not in blacklist:
metric_name = os.path.basename(gin_eval_config).replace(
".gin", "")
print("Computing metric " + metric_name + " on " + post_name +
"...")
metric_dir = os.path.join(output_directory, "metrics",
post_name, metric_name)
eval_bindings = [
"evaluation.random_seed = {}".format(
random_state.randint(2**32)),
"evaluation.name = '{}'".format(metric_name)
]
evaluate.evaluate_with_gin(post_dir, metric_dir,
args.overwrite, [gin_eval_config],
eval_bindings)
# We visualize reconstructions, samples and latent space traversals.
visualize_dir = os.path.join(output_directory, "visualizations")
visualize_model.visualize(model_dir, visualize_dir, args.overwrite)
def main():
parser = argparse.ArgumentParser(description='Project description.')
parser.add_argument('--study', help='Name of the study.', type=str, default='unsupervised_study_v1')
parser.add_argument('--output_directory', help='Output directory of experiments.', type=str, default=None)
parser.add_argument('--model_dir', help='Directory to take trained model from.', type=str, default=None)
parser.add_argument('--model_num', help='Integer with model number to train.', type=int, default=None)
parser.add_argument('--only_print', help='Whether to only print the hyperparameter settings.', type=_str_to_bool, default=False)
parser.add_argument('--overwrite', help='Whether to overwrite output directory.', type=_str_to_bool, default=False)
args = parser.parse_args()
# logging.set_verbosity('error')
# logging.set_stderrthreshold('error')
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
# Obtain the study to reproduce.
study = reproduce.STUDIES[args.study]
# Print the hyperparameter settings.
if args.model_dir is None:
study.print_model_config(args.model_num)
else:
print("Model directory (skipped training):")
print("--")
print(args.model_dir)
print()
study.print_postprocess_config()
print()
study.print_eval_config()
if args.only_print:
return
# Set correct output directory.
if args.output_directory is None:
if args.model_dir is None:
output_directory = os.path.join("output", "{study}", "{model_num}")
else:
output_directory = "output"
else:
output_directory = args.output_directory
# Insert model number and study name into path if necessary.
output_directory = output_directory.format(model_num=str(args.model_num),
study=str(args.study))
# Model training (if model directory is not provided).
if args.model_dir is None:
model_bindings, model_config_file = study.get_model_config(args.model_num)
print("Training model...")
model_dir = os.path.join(output_directory, "model")
model_bindings = [
"model.name = '{}'".format(os.path.basename(model_config_file)).replace(
".gin", ""),
"model.model_num = {}".format(args.model_num),
] + model_bindings
train.train_with_gin(model_dir, args.overwrite, [model_config_file],
model_bindings)
else:
print("Skipped training...")
model_dir = args.model_dir
# We visualize reconstructions, samples and latent space traversals.
visualize_dir = os.path.join(output_directory, "visualizations")
visualize_model.visualize(model_dir, visualize_dir, args.overwrite)
# We fix the random seed for the postprocessing and evaluation steps (each
# config gets a different but reproducible seed derived from a master seed of
# 0). The model seed was set via the gin bindings and configs of the study.
random_state = np.random.RandomState(0)
# We extract the different representations and save them to disk.
postprocess_config_files = sorted(study.get_postprocess_config_files())
for config in postprocess_config_files:
post_name = os.path.basename(config).replace(".gin", "")
print("Extracting representation %s..." % post_name)
post_dir = os.path.join(output_directory, "postprocessed", post_name)
postprocess_bindings = [
"postprocess.random_seed = {}".format(random_state.randint(2**32)),
"postprocess.name = '{}'".format(post_name)
]
postprocess.postprocess_with_gin(model_dir, post_dir, args.overwrite,
[config], postprocess_bindings)
# Iterate through the disentanglement metrics.
eval_configs = sorted(study.get_eval_config_files())
blacklist = ['downstream_task_logistic_regression.gin']
for config in postprocess_config_files:
post_name = os.path.basename(config).replace(".gin", "")
post_dir = os.path.join(output_directory, "postprocessed",
post_name)
# Now, we compute all the specified scores.
for gin_eval_config in eval_configs:
if os.path.basename(gin_eval_config) not in blacklist:
metric_name = os.path.basename(gin_eval_config).replace(".gin", "")
print("Computing metric '%s' on '%s'..." % (metric_name, post_name))
metric_dir = os.path.join(output_directory, "metrics", post_name,
metric_name)
eval_bindings = [
"evaluation.random_seed = {}".format(random_state.randint(2**32)),
"evaluation.name = '{}'".format(metric_name)
]
evaluate.evaluate_with_gin(post_dir, metric_dir, args.overwrite,
[gin_eval_config], eval_bindings)
