def sample_mlp_vae_family_cfg(seed):
"""Samples a task config for a MLP VAE model on image datasets.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
rng = np.random.RandomState(seed)
cfg = {}
enc_n_layers = rng.choice([1, 2, 3, 4])
cfg["enc_hidden_units"] = [
utils.sample_log_int(rng, 32, 128) for _ in range(enc_n_layers)
]
dec_n_layers = rng.choice([1, 2, 3])
cfg["dec_hidden_units"] = [
utils.sample_log_int(rng, 32, 128) for _ in range(dec_n_layers)
]
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_image_dataset(rng)
return cfg
def sample_rnn_text_classification_family_cfg(seed):
"""Samples a task config for an RNN classification task.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
rng = np.random.RandomState(seed)
cfg = {}
cfg["embed_dim"] = utils.sample_log_int(rng, 8, 128)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_text_dataset(rng)
# TODO(lmetz) trim this if using characters...
cfg["vocab_size"] = utils.sample_log_int(rng, 100, 10000)
cfg["core"] = utils.sample_rnn_core(rng)
cfg["trainable_init"] = bool(rng.choice([True, False]))
cfg["loss_compute"] = rng.choice(["last", "avg", "max"])
return cfg
def sample_mlp_ae_family_cfg(seed):
"""Sample a task config for a mlp autoencoder on image data.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
rng = np.random.RandomState(seed)
cfg = {}
n_layers = rng.choice([1, 2, 3, 4, 5, 6])
cfg["hidden_units"] = [
utils.sample_log_int(rng, 32, 128) for _ in range(n_layers)
]
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_image_dataset(rng)
# Give relu double weight as this is what is often used in practice.
cfg["output_type"] = rng.choice(
["tanh", "tanh", "sigmoid", "sigmoid", "linear_center", "linear"])
# Give l2 double weight as this is often used
cfg["loss_type"] = rng.choice(["l2", "l2", "l1"])
cfg["reduction_type"] = rng.choice(["reduce_mean", "reduce_sum"])
return cfg
def sample_mlp_family_cfg(seed):
"""Sample a task config for a MLP model on image datasets.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
# random offset seed to ensure different task families sample different
# configs.
rng = np.random.RandomState(seed + 99123491)
cfg = collections.OrderedDict()
n_layers = rng.choice([1, 2, 3, 4, 5, 6])
cfg["layer_sizes"] = [
utils.sample_log_int(rng, 16, 128) for _ in range(n_layers)
]
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_image_dataset(rng)
cfg["center_data"] = bool(rng.choice([True, False]))
return cfg
def sample_maf_family_cfg(seed):
"""Sample a task config for a MAF model on image datasets.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
# random offset to seed to prevent clashing with other problems.
rng = np.random.RandomState(seed + 6123348)
cfg = {}
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_image_dataset(rng)
n_layers = int(rng.choice([1, 2]))
cfg["hidden_units"] = [
utils.sample_log_int(rng, 16, 128) for _ in range(n_layers)
]
cfg["num_bijectors"] = int(rng.choice([1, 2, 3, 4]))
return cfg
def sample_nvp_family_cfg(seed):
"""Sample a task config for a NVP model on image datasets.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
# Random offset on seed to ensure different randomness for task.
rng = np.random.RandomState(seed + 123919936)
cfg = {}
# TODO(lmetz) consider also forwarding per layer activation and per layer
# w_init.
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["dataset"] = utils.sample_image_dataset(rng)
n_layers = int(rng.choice([1, 2]))
cfg["hidden_units"] = [
utils.sample_log_int(rng, 16, 128) for _ in range(n_layers)
]
cfg["num_bijectors"] = int(rng.choice([1, 2, 3, 4]))
return cfg
def sample_word_language_model_family_cfg(seed):
"""Sample a word language model config."""
rng = np.random.RandomState(seed)
cfg = {}
cfg["embed_dim"] = utils.sample_log_int(rng, 8, 128)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["vocab_size"] = utils.sample_log_int(rng, 1000, 30000)
cfg["core"] = utils.sample_rnn_core(rng)
cfg["trainable_init"] = bool(rng.choice([True, False]))
cfg["dataset"] = utils.sample_word_lm_dataset(rng)
return cfg
def sample_conv_fc_family_cfg(seed):
"""Sample a task config for a conv net with a fully connected layer on top.
These configs are nested python structures that provide enough information
to create an instance of the problem.
Args:
seed: int Random seed to generate task from.
Returns:
A nested dictionary containing a configuration.
"""
rng = np.random.RandomState(seed)
cfg = {}
layer_choices = [1, 2, 3, 4, 5]
max_layer = np.max(layer_choices)
n_layers = rng.choice(layer_choices)
# pattern for how strides are chosen. Either all 2s, repeated 1,2
# or repeated 2,1.
stride_pattern = rng.choice(["all_two", "one_two", "two_one"])
if stride_pattern == "all_two":
cfg["strides"] = ([2] * max_layer)[0:n_layers]
elif stride_pattern == "one_two":
cfg["strides"] = ([1, 2] * max_layer)[0:n_layers]
elif stride_pattern == "two_one":
cfg["strides"] = ([2, 1] * max_layer)[0:n_layers]
cfg["strides"] = list(zip(cfg["strides"], cfg["strides"]))
cfg["hidden_units"] = [
utils.sample_log_int(rng, 8, 64) for _ in range(n_layers)
]
cfg["activation"] = utils.sample_activation(rng)
cfg["w_init"] = utils.sample_initializer(rng)
cfg["padding"] = [
str(rng.choice([snt.SAME, snt.VALID])) for _ in range(n_layers)
]
n_fc_layers = rng.choice([0, 1, 2, 3])
cfg["fc_hidden_units"] = [
utils.sample_log_int(rng, 32, 128) for _ in range(n_fc_layers)
]
cfg["use_bias"] = bool(rng.choice([True, False]))
cfg["dataset"] = utils.sample_image_dataset(rng)
cfg["center_data"] = bool(rng.choice([True, False]))
return cfg
def test_sample_get_initializer(self):
rng = np.random.RandomState(123)
sampled_init = []
num = 3000
for _ in range(num):
init_name, args = utils.sample_initializer(rng)
sampled_init.append(init_name)
# smoke test to ensure graph builds
out = utils.get_initializer((init_name, args))((10, 10))
self.assertIsInstance(out, tf.Tensor)
uniques, counts = np.unique(sampled_init, return_counts=True)
counts_map = {str(u): c for u, c in zip(uniques, counts)}
# 13 is the total sum of unnormalized probs
amount_per_n = num / float(13)
self.assertNear(counts_map["he_normal"], amount_per_n * 2, 40)
self.assertNear(counts_map["orthogonal"], amount_per_n, 40)
self.assertNear(counts_map["glorot_normal"], amount_per_n * 2, 40)
def sample_char_rnn_language_model_family_cfg(seed):
"""Samples a character NN language modeling task."""
rng = np.random.RandomState(seed)
cfg = {}
cfg["embed_dim"] = utils.sample_log_int(rng, 8, 128)
cfg["w_init"] = utils.sample_initializer(rng)
full_vocab = utils.sample_bool(rng, 0.8)
if full_vocab:
cfg["vocab_size"] = 256
else:
# only operate on some subset of full words.
cfg["vocab_size"] = utils.sample_log_int(rng, 100, 256)
cfg["core"] = utils.sample_rnn_core(rng)
cfg["trainable_init"] = bool(rng.choice([True, False]))
cfg["dataset"] = utils.sample_char_lm_dataset(rng)
return cfg
