def tuning(self):
self.HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([3, 6]))
self.HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2))
self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam',
'sgd']))
self.METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[
self.HP_NUM_UNITS, self.HP_DROPOUT, self.HP_OPTIMIZER
],
metrics=[
hp.Metric(self.METRIC_ACCURACY, display_name='Accuracy')
],
)
session_num = 0
for num_units in self.HP_NUM_UNITS.domain.values:
for dropout_rate in (self.HP_DROPOUT.domain.min_value,
self.HP_DROPOUT.domain.max_value):
for optimizer in self.HP_OPTIMIZER.domain.values:
hparams = {
self.HP_NUM_UNITS: num_units,
self.HP_DROPOUT: dropout_rate,
self.HP_OPTIMIZER: optimizer,
}
run_name = "run-%d" % session_num
print('--- Starting trial: %s' % run_name)
print({h.name: hparams[h] for h in hparams})
self.run('logs/hparam_tuning/' + run_name, hparams)
session_num += 1
def evaluate(self, avpool=True, query_fig=False):
if self._test:
test_recon_loss = 0
for x in self._test_ds:
reconstructed = self._models["full"](x)
test_recon_loss += np.mean(
np.abs(x.numpy() - reconstructed.numpy()))
self._record_scalars(test_reconstruction_loss=test_recon_loss /
self._test_steps)
test_ims = np.concatenate([x.numpy(), reconstructed.numpy()], 2)
self._record_images(test_images=test_ims)
if self._downstream_labels is not None:
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
hparams = {
hp.HParam("dropout", hp.RealInterval(0., 1.)):
self.config["dropout"]
}
else:
hparams = None
self._linear_classification_test(hparams,
avpool=avpool,
query_fig=query_fig)
def _create_hyprparameters_domain():
HP_LEARNING_RATE = hp.HParam(
"learning_rate",
hp.RealInterval(1.0e-4, 7.0e-3),
)
HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9))
HP_LEARNING_RATE_DECAY_STEPS = hp.HParam(
"learning_rate_decay_steps", hp.Discrete(range(1_000, 10_000 + 1)))
HP_WEIGHT_DECAY = hp.HParam("weight_decay",
hp.RealInterval(1.0e-4, 1.0e-3))
return [
HP_LEARNING_RATE,
HP_BETA_1,
HP_LEARNING_RATE_DECAY_STEPS,
HP_WEIGHT_DECAY,
]
def _add_distributions(
self,
distributions: Dict[str,
optuna.distributions.BaseDistribution]) -> None:
for param_name, param_distribution in distributions.items():
if isinstance(param_distribution,
optuna.distributions.UniformDistribution):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.RealInterval(param_distribution.low,
param_distribution.high))
elif isinstance(param_distribution,
optuna.distributions.LogUniformDistribution):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.RealInterval(param_distribution.low,
param_distribution.high))
elif isinstance(param_distribution,
optuna.distributions.DiscreteUniformDistribution):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.Discrete(param_distribution.low,
param_distribution.high))
elif isinstance(param_distribution,
optuna.distributions.IntUniformDistribution):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.IntInterval(param_distribution.low,
param_distribution.high))
elif isinstance(param_distribution,
optuna.distributions.CategoricalDistribution):
self._hp_params[param_name] = hp.HParam(
param_name, hp.Discrete(param_distribution.choices))
else:
distribution_list = [
optuna.distributions.UniformDistribution.__name__,
optuna.distributions.LogUniformDistribution.__name__,
optuna.distributions.DiscreteUniformDistribution.__name__,
optuna.distributions.IntUniformDistribution.__name__,
optuna.distributions.CategoricalDistribution.__name__,
]
raise NotImplementedError(
"The distribution {} is not implemented. "
"The parameter distribution should be one of the {}".
format(param_distribution, distribution_list))
def _create_hyprparameters_domain():
HP_LEARNING_RATE = hp.HParam(
"learning_rate",
hp.RealInterval(1.0e-3, 7.0e-3),
)
HP_LEARNING_RATE_DECAY_STEPS = hp.HParam(
"learning_rate_decay_steps",
hp.Discrete(range(1_000, 6_000 + 1)),
)
HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9))
HP_FR_WEIGHT = hp.HParam("face_recognition_weight",
hp.RealInterval(0.1, 1.0))
HP_SR_WEIGHT = hp.HParam("super_resolution_weight",
hp.RealInterval(0.1, 1.0))
HP_PERCEPTUAL_WEIGHT = hp.HParam("perceptual_weight",
hp.RealInterval(1.0e-3, 1.0e-2))
HP_GENERATOR_WEIGHT = hp.HParam("generator_weight",
hp.RealInterval(1.0e-2, 7.0e-2))
HP_L1_WEIGHT = hp.HParam("l1_weight", hp.RealInterval(1.0e-2, 9.0e-2))
return [
HP_LEARNING_RATE,
HP_LEARNING_RATE_DECAY_STEPS,
HP_BETA_1,
HP_FR_WEIGHT,
HP_SR_WEIGHT,
HP_PERCEPTUAL_WEIGHT,
HP_GENERATOR_WEIGHT,
HP_L1_WEIGHT,
]
def evaluate(self):
b = tf.expand_dims(tf.expand_dims(self._buffer,0),-1)
self._record_images(buffer=b)
if self._downstream_labels is not None:
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
hparams = {
hp.HParam("tau", hp.RealInterval(0., 10000.)):self.config["tau"],
hp.HParam("alpha", hp.RealInterval(0., 1.)):self.config["alpha"],
hp.HParam("batches_in_buffer", hp.IntInterval(1, 1000000)):self.config["batches_in_buffer"],
hp.HParam("output_dim", hp.IntInterval(1, 1000000)):self.config["output_dim"],
hp.HParam("sobel", hp.Discrete([True, False])):self.input_config["sobel"]
}
else:
hparams=None
self._linear_classification_test(hparams)
def log_hyperparameters():
"""
Blueprint for hyperparameter and metric logging in tensorboard during hyperparameter tuning
Returns:
logparams (list): List containing the hyperparameters to log in tensorboard.
metrics (list): List containing the metrics to log in tensorboard.
"""
logparams = [
hp.HParam(
"latent_dim",
hp.Discrete([2, 4, 6, 8, 12, 16]),
display_name="latent_dim",
description="encoding size dimensionality",
),
hp.HParam(
"n_components",
hp.IntInterval(min_value=1, max_value=25),
display_name="n_components",
description="latent component number",
),
hp.HParam(
"gram_weight",
hp.RealInterval(min_value=0.0, max_value=1.0),
display_name="gram_weight",
description="weight of the gram loss",
),
]
metrics = [
hp.Metric(
"val_number_of_populated_clusters",
display_name="number of populated clusters",
),
hp.Metric(
"val_reconstruction_loss",
display_name="reconstruction loss",
),
hp.Metric(
"val_gram_loss",
display_name="gram loss",
),
hp.Metric(
"val_vq_loss",
display_name="vq loss",
),
hp.Metric(
"val_total_loss",
display_name="total loss",
),
]
return logparams, metrics
def _create_hyprparameters_domain():
HP_LEARNING_RATE = hp.HParam(
"learning_rate",
hp.RealInterval(1.0e-3, 7.0e-3),
)
HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9))
HP_PERCEPTUAL_WEIGHT = hp.HParam("perceptual_weight",
hp.RealInterval(1.0e-3, 1.0))
HP_GENERATOR_WEIGHT = hp.HParam("generator_weight",
hp.RealInterval(1.0e-3, 7.0e-2))
HP_L1_WEIGHT = hp.HParam("l1_weight", hp.RealInterval(1.0e-2, 9.0e-2))
return [
HP_LEARNING_RATE,
HP_BETA_1,
HP_PERCEPTUAL_WEIGHT,
HP_GENERATOR_WEIGHT,
HP_L1_WEIGHT,
]
def __init__(self, lr_range=(1.e-8, 1.), momentum_range=(0., 1.)):
super(SGDMomentumManager, self).__init__(lr_range=lr_range)
self._hparam['momentum'] = hp.HParam(
'momentum',
description="Momentum Rate",
domain=hp.RealInterval(*momentum_range))
self._hparam['nesterov'] = hp.HParam('nesterov',
description="Momentum",
domain=hp.Discrete(
['standard', 'nesterov']))
def convert_hyperparams_to_hparams(
hyperparams: hp_module.HyperParameters
) -> Dict[hparams_api.HParam, Any]:
"""Converts KerasTuner HyperParameters to TensorBoard HParams.
Args:
hyperparams: A KerasTuner HyperParameters instance
Returns:
A dict that maps TensorBoard HParams to current values.
"""
hparams = {}
for hp in hyperparams.space:
hparams_value = {}
try:
hparams_value = hyperparams.get(hp.name)
except ValueError:
continue
hparams_domain = {}
if isinstance(hp, hp_module.Choice):
hparams_domain = hparams_api.Discrete(hp.values)
elif isinstance(hp, hp_module.Int):
if hp.step is None or hp.step == 1:
hparams_domain = hparams_api.IntInterval(
hp.min_value, hp.max_value)
else:
# Note: `hp.max_value` is inclusive, unlike the end index
# of Python `range()`, which is exclusive
values = list(range(hp.min_value, hp.max_value + 1, hp.step))
hparams_domain = hparams_api.Discrete(values)
elif isinstance(hp, hp_module.Float):
if hp.step is None:
hparams_domain = hparams_api.RealInterval(
hp.min_value, hp.max_value)
else:
# Note: `hp.max_value` is inclusive, which is also
# the default for Numpy's linspace
num_samples = int((hp.max_value - hp.min_value) / hp.step)
end_value = hp.min_value + (num_samples * hp.step)
values = np.linspace(hp.min_value, end_value,
num_samples + 1).tolist()
hparams_domain = hparams_api.Discrete(values)
elif isinstance(hp, hp_module.Boolean):
hparams_domain = hparams_api.Discrete([True, False])
elif isinstance(hp, hp_module.Fixed):
hparams_domain = hparams_api.Discrete([hp.value])
else:
raise ValueError(
"`HyperParameter` type not recognized: {}".format(hp))
hparams_key = hparams_api.HParam(hp.name, hparams_domain)
hparams[hparams_key] = hparams_value
return hparams
def evaluate(self, avpool=True):
predictions = self._models["full"].predict(self._val_ds)
num_categories = len(self.categories)
for i in range(num_categories):
category = self.categories[i]
preds = predictions[:, i]
y_true = self._val_labels[:, i]
acc = np.mean(y_true == (preds >= 0.5).astype(int))
auc = roc_auc_score(y_true, preds)
self._record_scalars(**{
f"val_accuracy_{category}": acc,
f"val_auc_{category}": auc
},
metric=True)
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
metrics = []
for c in self.categories:
metrics.append(f"val_accuracy_{c}")
metrics.append(f"val_auc_{c}")
hparams = {
hp.HParam("lam", hp.RealInterval(0., 10000.)):
self.config["lam"],
hp.HParam("tau", hp.RealInterval(0., 10000.)):
self.config["tau"],
hp.HParam("mu", hp.RealInterval(0., 10000.)):
self.config["mu"],
hp.HParam("batch_size", hp.RealInterval(0., 10000.)):
self.input_config["batch_size"],
hp.HParam("lr", hp.RealInterval(0., 10000.)):
self.config["lr"],
hp.HParam("lr_decay", hp.RealInterval(0., 10000.)):
self.config["lr_decay"],
hp.HParam("decay_type",
hp.Discrete(["cosine", "exponential", "staircase"])):
self.config["decay_type"],
hp.HParam("opt_type", hp.Discrete(["sgd", "adam", "momentum"])):
self.config["opt_type"],
hp.HParam("weight_decay", hp.RealInterval(0., 10000.)):
self.config["weight_decay"]
}
self._hparams_config = hp.hparams_config(
hparams=list(hparams.keys()),
metrics=[hp.Metric(m) for m in metrics])
# record hyperparameters
base_dir, run_name = os.path.split(self.logdir)
if len(run_name) == 0:
base_dir, run_name = os.path.split(base_dir)
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
self.onBindingUI()
self.batch_size = 32
self.learning_rate = 0.001
self.optimizer = 'sgd'
self.HP_BATCH_SIZE = hp.HParam('num_units', hp.Discrete([self.batch_size, self.batch_size]))
self.HP_LEARNING_RATE = hp.HParam('dropout', hp.RealInterval(self.learning_rate, self.learning_rate))
self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete([self.optimizer]))
def __init__(self, lr_range=(1.e-8, 1.)):
"""Create a SGD manager with a specific learning rate range"""
super(SGDManager, self).__init__()
self._hparam = {
"optimizer":
hp.HParam("optimizer",
domain=hp.Discrete(['SGD']),
display_name="Optimizer"),
"learning_rate":
hp.HParam("learning_rate",
domain=hp.RealInterval(*lr_range),
display_name="Learning rate"),
}
def evaluate(self):
num_test_images = 10
if self._test:
preds = self._models["inpainter"].predict(self._test_masked_ims)
preds = preds[:, :self.input_config["imshape"][0], :self.
input_config["imshape"][1], :]
reconstruction_residual = self._test_mask * (preds -
self._test_ims)
reconstructed_loss = np.mean(np.abs(reconstruction_residual))
# see how the discriminator does on them
disc_outputs_on_raw = self._models["discriminator"].predict(
self._test_ims)
disc_outputs_on_inpaint = self._models["discriminator"].predict(
preds)
# for the visualization in tensorboard: replace the unmasked areas
# with the input image as a guide to the eye
preds = preds * self._test_mask + self._test_ims * (
1 - self._test_mask)
predviz = np.concatenate([
self._test_masked_ims[:num_test_images],
preds[:num_test_images]
], 2).astype(np.float32)
# record all the summaries
tf.summary.image("inpaints",
predviz,
step=self.step,
max_outputs=10)
tf.summary.histogram("disc_outputs_on_raw",
disc_outputs_on_raw,
step=self.step)
tf.summary.histogram("disc_outputs_on_inpaint",
disc_outputs_on_inpaint,
step=self.step)
self._record_scalars(test_recon_loss=reconstructed_loss)
if self._downstream_labels is not None:
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
hparams = {
hp.HParam("adv_weight", hp.RealInterval(0., 10000.)):
self.config["adv_weight"],
hp.HParam("sobel", hp.Discrete([True, False])):
self.input_config["sobel"]
}
else:
hparams = None
self._linear_classification_test(hparams)
def kt_to_hparam(hp: kt.HyperParameter) -> hp_lib.HParam:
if isinstance(hp, kt.engine.hyperparameters.Float):
domain = hp_lib.RealInterval(hp.min_value, hp.max_value)
elif isinstance(hp, kt.engine.hyperparameters.Int):
domain = hp_lib.IntInterval(hp.min_value, hp.max_value)
elif isinstance(hp, kt.engine.hyperparameters.Boolean):
domain = hp_lib.Discrete([False, True], dtype=bool)
elif isinstance(hp, kt.engine.hyperparameters.Fixed):
domain = hp_lib.Discrete([hp.value])
elif isinstance(hp, kt.engine.hyperparameters.Choice):
domain = hp_lib.Discrete(hp.values)
else:
raise TypeError(f"Unsupposed hyperparamter type {hp}")
return hp_lib.HParam(hp.name, domain)
def __init__(self,
lr_range=(1.e-8, 1.),
beta_1_range=(1. - 1.e-1, 1. - 1.e-4),
beta_2_range=(1. - 1.e-3, 1. - 1e-6),
epsilon_range=(1.e-8, 1.e-3)):
"""Create an Adam/Adamax manager with a specific range for hyperparameters
A last non-tunable hyperparameter is `optimizer`, which can be Adam, Amsgrad, Adamax.
Args:
lr_range ():
beta_1_range ():
beta_2_range ():
epsilon_range ():
"""
super(AdamManager, self).__init__()
self._hparam = {
"optimizer":
hp.HParam("optimizer",
domain=hp.Discrete(['Adam', 'Amsgrad', 'Adamax']),
display_name="Optimizer"),
"learning_rate":
hp.HParam("learning_rate",
domain=hp.RealInterval(*lr_range),
display_name="Learning rate"),
"beta_1":
hp.HParam("beta_1",
domain=hp.RealInterval(*beta_1_range),
display_name="Beta1"),
"beta_2":
hp.HParam("beta_2",
domain=hp.RealInterval(*beta_2_range),
display_name="Beta2"),
"epsilon":
hp.HParam("epsilon",
domain=hp.RealInterval(*epsilon_range),
display_name="Epsilon"),
}
def evaluate(self):
if self._downstream_labels is not None:
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
hparams = {
hp.HParam("temperature", hp.RealInterval(0., 10000.)):
self.config["temperature"],
hp.HParam("num_hidden", hp.IntInterval(1, 1000000)):
self.config["num_hidden"],
hp.HParam("output_dim", hp.IntInterval(1, 1000000)):
self.config["output_dim"]
}
else:
hparams = None
self._linear_classification_test(hparams)
def prepare_hparams(self, hparams_domains):
"""Convert informal specifications to a dict of hp.HParam"""
domains = {}
domains["num_layers"] = hp.HParam(
"num_layers",
hp.Discrete(hparams_domains["num_layers"]))
domains["learning_rate"] = hp.HParam(
"learning_rate",
hp.RealInterval(*hparams_domains["learning_rate"]))
domains["num_filters"] = hp.HParam(
"num_filters",
hp.Discrete(hparams_domains["num_filters"]))
domains["batch_normalization"] = hp.HParam(
"batch_normalization",
hp.Discrete(hparams_domains["batch_normalization"]))
return domains
def _add_distributions(
self,
distributions: Dict[str,
optuna.distributions.BaseDistribution]) -> None:
real_distributions = (
optuna.distributions.UniformDistribution,
optuna.distributions.LogUniformDistribution,
optuna.distributions.DiscreteUniformDistribution,
optuna.distributions.FloatDistribution,
)
int_distributions = (
optuna.distributions.IntUniformDistribution,
optuna.distributions.IntLogUniformDistribution,
optuna.distributions.IntDistribution,
)
categorical_distributions = (
optuna.distributions.CategoricalDistribution, )
supported_distributions = (real_distributions + int_distributions +
categorical_distributions)
for param_name, param_distribution in distributions.items():
if isinstance(param_distribution, real_distributions):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.RealInterval(float(param_distribution.low),
float(param_distribution.high)),
)
elif isinstance(param_distribution, int_distributions):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.IntInterval(param_distribution.low,
param_distribution.high),
)
elif isinstance(param_distribution, categorical_distributions):
self._hp_params[param_name] = hp.HParam(
param_name,
hp.Discrete(param_distribution.choices),
)
else:
distribution_list = [
distribution.__name__
for distribution in supported_distributions
]
raise NotImplementedError(
"The distribution {} is not implemented. "
"The parameter distribution should be one of the {}".
format(param_distribution, distribution_list))
def convert_hyperparams_to_hparams(hyperparams):
"""Converts KerasTuner HyperParameters to TensorBoard HParams."""
hparams = {}
for hp in hyperparams.space:
hparams_value = {}
try:
hparams_value = hyperparams.get(hp.name)
except ValueError:
continue
hparams_domain = {}
if isinstance(hp, hp_module.Choice):
hparams_domain = hparams_api.Discrete(hp.values)
elif isinstance(hp, hp_module.Int):
if hp.step is not None and hp.step != 1:
# Note: `hp.max_value` is inclusive, unlike the end index
# of Python `range()`, which is exclusive
values = list(range(hp.min_value, hp.max_value + 1, hp.step))
hparams_domain = hparams_api.Discrete(values)
else:
hparams_domain = hparams_api.IntInterval(
hp.min_value, hp.max_value)
elif isinstance(hp, hp_module.Float):
if hp.step is not None:
# Note: `hp.max_value` is inclusive, unlike the end index
# of Numpy's arange(), which is exclusive
values = np.arange(hp.min_value,
hp.max_value + 1e-7,
step=hp.step).tolist()
hparams_domain = hparams_api.Discrete(values)
else:
hparams_domain = hparams_api.RealInterval(
hp.min_value, hp.max_value)
elif isinstance(hp, hp_module.Boolean):
hparams_domain = hparams_api.Discrete([True, False])
elif isinstance(hp, hp_module.Fixed):
hparams_domain = hparams_api.Discrete([hp.value])
else:
raise ValueError(
"`HyperParameter` type not recognized: {}".format(hp))
hparams_key = hparams_api.HParam(hp.name, hparams_domain)
hparams[hparams_key] = hparams_value
return hparams
def __init__(self,
num_session_groups=10,
HP_CONV_LAYERS=hp.HParam("conv_layers", hp.IntInterval(1, 3)),
HP_CONV_KERNEL_SIZE=hp.HParam("conv_kernel_size", hp.Discrete([3, 5])),
HP_DENSE_LAYERS=hp.HParam("dense_layers", hp.IntInterval(1, 3)),
HP_DROPOUT=hp.HParam("dropout", hp.RealInterval(0.1, 0.4)),
HP_OPTIMIZER=hp.HParam("optimizer", hp.Discrete(["adam", "adagrad"]))):
self.HP_CONV_LAYERS = HP_CONV_LAYERS
self.HP_CONV_KERNEL_SIZE = HP_CONV_KERNEL_SIZE
self.HP_DENSE_LAYERS = HP_DENSE_LAYERS
self.HP_DROPOUT = HP_DROPOUT
self.HP_OPTIMIZER = HP_OPTIMIZER
self.num_session_groups = num_session_groups
self.HPARAMS = [
HP_CONV_LAYERS,
HP_CONV_KERNEL_SIZE,
HP_DENSE_LAYERS,
HP_DROPOUT,
HP_OPTIMIZER,
]
self.METRICS = [
hp.Metric(
"epoch_accuracy",
group="train",
display_name="accuracy (train)",
),
hp.Metric(
"epoch_loss",
group="train",
display_name="loss (train)",
),
hp.Metric(
"epoch_accuracy",
group="validation",
display_name="accuracy (val.)",
),
hp.Metric(
"epoch_loss",
group="validation",
display_name="loss (val.)",
)
]
def hparam_from_config_space_item(self, c_item):
name = c_item['config']['name']
p_type = c_item['class_name']
cfg = c_item['config']
# parm = hp.Discrete()
hparam = None
if p_type == 'Int':
hparam = hp.HParam(name=name,
domain=hp.IntInterval(cfg['min_value'],
cfg['max_value']))
elif p_type == 'Float':
hparam = hp.HParam(name=name,
domain=hp.RealInterval(cfg['min_value'],
cfg['max_value']))
elif p_type == 'Fixed':
hparam = hp.HParam(name=name, domain=hp.Discrete([cfg['value']]))
elif p_type == 'Choice':
hparam = hp.HParam(name=name, domain=hp.Discrete(cfg['values']))
return hparam
def evaluate(self):
if self._test:
# compute features- in memory for now
f_x = []
f_y = []
for x, y in self._test_ds:
f_x.append(self.flatten(self.fcn(x)))
f_y.append(self.flatten(self.fcn(y)))
f_x = tf.concat(f_x, 0)
f_y = tf.concat(f_y, 0)
# compute P for each head
for e, h in enumerate(self.heads):
P = compute_p(f_x, f_y, h)
self._record_images(**{"P_head_%s"%e:P})
for e, h in enumerate(self.heads_oc):
P = compute_p(f_x, f_y, h)
self._record_images(**{"P_oc_head_%s"%e:P})
if self._downstream_labels is not None:
# choose the hyperparameters to record
if not hasattr(self, "_hparams_config"):
from tensorboard.plugins.hparams import api as hp
hparams = {
hp.HParam("k", hp.IntInterval(0, 1000000),
description="output dimension for clustering head"):self.config["k"],
hp.HParam("h", hp.IntInterval(0, 1000000),
description="number of clustering sub-heads"):self.config["h"],
hp.HParam("k_oc", hp.IntInterval(0, 1000000),
description="output dimension for overclustering head"):self.config["k_oc"],
hp.HParam("entropy_weight", hp.RealInterval(0., 1000000.),
description="additional weight factor for entropy in loss function"):self.config["entropy_weight"],
hp.HParam("r", hp.IntInterval(0, 1000000),
description="number of times to repeat each image sequentially in the data pipeline"):self.config["r"],
hp.HParam("sobel", hp.Discrete([True, False]),
description="whether Sobel filtering was applied to inputs"):self.input_config["sobel"]
}
else:
hparams=None
self._linear_classification_test(hparams)
def __init__(self,
*,
input_size,
n_classes,
activations=("relu", ),
losses=("categorical_crossentropy", )):
self.input_size = input_size
self.n_classes = n_classes
self._hparam = {
"n_layers":
hp.HParam("n_layers",
domain=hp.IntInterval(1, 100),
display_name="Depth"),
"layer_size":
hp.HParam("layer_size",
domain=hp.IntInterval(1, 100),
display_name="Width"),
"layer_activation":
hp.HParam("layer_activation",
domain=hp.Discrete(activations),
display_name="Activation Fct."),
"reg":
hp.HParam("reg",
domain=hp.RealInterval(0., 1.),
display_name="Reg."),
"loss_function":
hp.HParam("loss_function",
domain=hp.Discrete(losses),
display_name="Loss Fct.")
}
self._metrics = {
"accuracy": hp.Metric("accuracy", display_name="Accuracy")
}
self._model = None
def _configure_hparams(logdir, dicts,
metrics=["linear_classification_accuracy",
"alignment", "uniformity"]):
"""
Set up the tensorboard hyperparameter interface
:logdir: string; path to log directory
:dicts: list of dictionaries containing hyperparameter values
:metrics: list of strings; metric names
"""
metrics = [hp.Metric(m) for m in metrics]
params = {}
# for each parameter dictionary
for d in dicts:
# for each parameter:
for k in d:
# is it a categorical?
if k in SPECIAL_HPARAMS:
params[hp.HParam(k, hp.Discrete(SPECIAL_HPARAMS[k]))] = d[k]
elif isinstance(d[k], bool):
params[hp.HParam(k, hp.Discrete([True, False]))] = d[k]
elif isinstance(d[k], int):
params[hp.HParam(k, hp.IntInterval(1, 1000000))] = d[k]
elif isinstance(d[k], float):
params[hp.HParam(k, hp.RealInterval(0., 10000000.))] = d[k]
#
hparams_config = hp.hparams_config(
hparams=list(params.keys()),
metrics=metrics)
# get a name for the run
base_dir, run_name = os.path.split(logdir)
if len(run_name) == 0:
base_dir, run_name = os.path.split(base_dir)
# record hyperparamers
hp.hparams(params, trial_id=run_name)
def train_and_evaluate(
model,
num_epochs,
steps_per_epoch,
train_data,
validation_steps,
eval_data,
output_dir,
n_steps_history,
FLAGS,
decay_type,
learning_rate=3e-5,
s=1,
n_batch_decay=1,
metric_accuracy='metric',
):
"""
Compiles keras model and loads data into it for training.
"""
logging.info('training the model ...')
model_callbacks = []
# create meta data dictionary
dict_model = {}
dict_data = {}
dict_parameter = {}
dict_hardware = {}
dict_results = {}
dict_type_job = {}
dict_software = {}
# for debugging only
activate_tensorboard = True
activate_hp_tensorboard = False # True
activate_lr = False
save_checkpoints = False # True
save_history_per_step = False # True
save_metadata = False # True
activate_timing = False # True
# drop official method that is not working
activate_tf_summary_hp = True # False
# hardcoded way of doing hp
activate_hardcoded_hp = True # True
# dependencies
if activate_tf_summary_hp:
save_history_per_step = True
if FLAGS.is_hyperparameter_tuning:
# get trial ID
suffix = mu.get_trial_id()
if suffix == '':
logging.error('No trial ID for hyper parameter job!')
FLAGS.is_hyperparameter_tuning = False
else:
# callback for hp
logging.info('Creating a callback to store the metric!')
if activate_tf_summary_hp:
hp_metric = mu.HP_metric(metric_accuracy)
model_callbacks.append(hp_metric)
if output_dir:
if activate_tensorboard:
# tensorflow callback
log_dir = os.path.join(output_dir, 'tensorboard')
if FLAGS.is_hyperparameter_tuning:
log_dir = os.path.join(log_dir, suffix)
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=log_dir,
histogram_freq=1,
embeddings_freq=0,
write_graph=True,
update_freq='batch',
profile_batch='10, 20')
model_callbacks.append(tensorboard_callback)
if save_checkpoints:
# checkpoints callback
checkpoint_dir = os.path.join(output_dir, 'checkpoint_model')
if not FLAGS.is_hyperparameter_tuning:
# not saving model during hyper parameter tuning
# heckpoint_dir = os.path.join(checkpoint_dir, suffix)
checkpoint_prefix = os.path.join(checkpoint_dir,
'ckpt_{epoch:02d}')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_prefix,
verbose=1,
save_weights_only=True)
model_callbacks.append(checkpoint_callback)
if activate_lr:
# decay learning rate callback
# code snippet to make the switching between different learning rate decays possible
if decay_type == 'exponential':
decay_fn = mu.exponential_decay(lr0=learning_rate, s=s)
elif decay_type == 'stepwise':
decay_fn = mu.step_decay(lr0=learning_rate, s=s)
elif decay_type == 'timebased':
decay_fn = mu.time_decay(lr0=learning_rate, s=s)
else:
decay_fn = mu.no_decay(lr0=learning_rate)
# exponential_decay_fn = mu.exponential_decay(lr0=learning_rate, s=s)
# lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn, verbose=1)
# model_callbacks.append(lr_scheduler)
# added these two lines for batch updates
lr_decay_batch = mu.LearningRateSchedulerPerBatch(decay_fn,
n_batch_decay,
verbose=1)
# lr_decay_batch = mu.LearningRateSchedulerPerBatch(exponential_decay_fn, n_batch_decay, verbose=0)
# lambda step: ((learning_rate - min_learning_rate) * decay_rate ** step + min_learning_rate))
model_callbacks.append(lr_decay_batch)
# print_lr = mu.PrintLR()
# model_callbacks.append(mu.PrintLR())
# ---------------------------------------------------------------------------------------------------------------
# callback to store all the learning rates
# all_learning_rates = mu.LearningRateSchedulerPerBatch(model.optimizer, n_steps_history)
# all_learning_rates = mu.LR_per_step()
# all_learning_rates = mu.LR_per_step(model.optimizer)
# model_callbacks.append(all_learning_rates) # disble
if save_history_per_step:
# callback to create history per step (not per epoch)
histories_per_step = mu.History_per_step(eval_data, n_steps_history)
model_callbacks.append(histories_per_step)
if activate_timing:
# callback to time each epoch
timing = mu.TimingCallback()
model_callbacks.append(timing)
# checking model callbacks for
logging.info('model\'s callback:\n {}'.format(str(model_callbacks)))
# train the model
# time the function
start_time = time.time()
logging.info('starting model.fit')
# verbose = 0 (silent)
# verbose = 1 (progress bar)
# verbose = 2 (one line per epoch)
verbose = 1
history = model.fit(train_data,
epochs=num_epochs,
steps_per_epoch=steps_per_epoch,
validation_data=eval_data,
validation_steps=validation_steps,
verbose=verbose,
callbacks=model_callbacks)
# print execution time
elapsed_time_secs = time.time() - start_time
logging.info('\nexecution time: {}'.format(
timedelta(seconds=round(elapsed_time_secs))))
# check model
logging.info('model summary ={}'.format(model.summary()))
logging.info('model input ={}'.format(model.inputs))
logging.info('model outputs ={}'.format(model.outputs))
# to be remove
logging.info('\ndebugging .... : ')
pp.print_info_data(train_data)
if activate_timing:
logging.info('timing per epoch:\n{}'.format(
list(
map(lambda x: str(timedelta(seconds=round(x))),
timing.timing_epoch))))
logging.info('timing per validation:\n{}'.format(
list(
map(lambda x: str(timedelta(seconds=round(x))),
timing.timing_valid))))
logging.info('sum timing over all epochs:\n{}'.format(
timedelta(seconds=round(sum(timing.timing_epoch)))))
# for hp parameter tuning in TensorBoard
if FLAGS.is_hyperparameter_tuning:
logging.info('setup hyperparameter tuning!')
# test
#params = json.loads(os.environ.get("CLUSTER_SPEC", "{}")).get("job", {})
#print('debug: CLUSTER_SPEC1:', params)
#params = json.loads(os.environ.get("CLUSTER_SPEC", "{}")).get("job", {}).get("job_args", {})
#print('debug: CLUSTER_SPEC2:', params)
logging.info('debug: os.environ.items():', os.environ.items())
#
if activate_hardcoded_hp:
# trick to bypass ai platform bug
logging.info('hardcoded hyperparameter tuning!')
value_accuracy = histories_per_step.accuracies[-1]
hpt = hypertune.HyperTune()
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag=metric_accuracy,
metric_value=value_accuracy,
global_step=0)
else:
# should be extracted from /var/hypertune/output.metric
logging.info('standard hyperparameter tuning!')
# is this needed ?
# value_accuracy = histories_per_step.accuracies[-1]
# look at the content of the file
path_metric = '/var/hypertune/output.metric'
logging.info('checking if /var/hypertune/output.metric exist!')
if os.path.isfile(path_metric):
logging.info('file {} exist !'.format(path_metric))
with open(path_metric, 'r') as f:
logging.info('content of output.metric: {}'.format(f.read()))
if activate_hp_tensorboard:
logging.info('setup TensorBoard for hyperparameter tuning!')
# CAIP
#params = json.loads(os.environ.get("TF_CONFIG", "{}")).get("job", {}).get("hyperparameters", {}).get("params", {})
#uCAIP
params = json.loads(
os.environ.get("CLUSTER_SPEC", "{}")
) #.get("job", {}).get("hyperparameters", {}).get("params", {})
print('debug: CLUSTER_SPEC:', params)
list_hp = []
hparams = {}
for el in params:
hp_dict = dict(el)
if hp_dict.get('type') == 'DOUBLE':
key_hp = hp.HParam(
hp_dict.get('parameter_name'),
hp.RealInterval(hp_dict.get('min_value'),
hp_dict.get('max_value')))
list_hp.append(key_hp)
try:
hparams[key_hp] = FLAGS[hp_dict.get(
'parameter_name')].value
except KeyError:
logging.error(
'hyperparameter key {} doesn\'t exist'.format(
hp_dict.get('parameter_name')))
hparams_dir = os.path.join(output_dir, 'hparams_tuning')
with tf.summary.create_file_writer(hparams_dir).as_default():
hp.hparams_config(
hparams=list_hp,
metrics=[
hp.Metric(metric_accuracy,
display_name=metric_accuracy)
],
)
hparams_dir = os.path.join(hparams_dir, suffix)
with tf.summary.create_file_writer(hparams_dir).as_default():
# record the values used in this trial
hp.hparams(hparams)
tf.summary.scalar(metric_accuracy, value_accuracy, step=1)
if save_history_per_step:
# save the history in a file
search = re.search('gs://(.*?)/(.*)', output_dir)
if search is not None:
# temp folder locally and to be ove on gcp later
history_dir = os.path.join('./', model.name)
os.makedirs(history_dir, exist_ok=True)
else:
# locally
history_dir = os.path.join(output_dir, model.name)
os.makedirs(history_dir, exist_ok=True)
logging.debug('history_dir: \n {}'.format(history_dir))
with open(history_dir + '/history', 'wb') as file:
model_history = mu.History_trained_model(history.history,
history.epoch,
history.params)
pickle.dump(model_history, file, pickle.HIGHEST_PROTOCOL)
with open(history_dir + '/history_per_step', 'wb') as file:
model_history_per_step = mu.History_per_steps_trained_model(
histories_per_step.steps,
histories_per_step.losses,
histories_per_step.accuracies,
histories_per_step.val_steps,
histories_per_step.val_losses,
histories_per_step.val_accuracies,
0, # all_learning_rates.all_lr,
0, # all_learning_rates.all_lr_alternative,
0) # all_learning_rates.all_lr_logs)
pickle.dump(model_history_per_step, file, pickle.HIGHEST_PROTOCOL)
if output_dir:
# save the model
savemodel_path = os.path.join(output_dir, 'saved_model')
if not FLAGS.is_hyperparameter_tuning:
# not saving model during hyper parameter tuning
# savemodel_path = os.path.join(savemodel_path, suffix)
model.save(os.path.join(savemodel_path, model.name))
model2 = tf.keras.models.load_model(
os.path.join(savemodel_path, model.name))
# check model
logging.info('model2 summary ={}'.format(model2.summary()))
logging.info('model2 input ={}'.format(model2.inputs))
logging.info('model2 outputs ={}'.format(model2.outputs))
logging.info('model2 signature outputs ={}'.format(
model2.signatures['serving_default'].structured_outputs))
logging.info('model2 inputs ={}'.format(
model2.signatures['serving_default'].inputs[0]))
if save_history_per_step:
# save history
search = re.search('gs://(.*?)/(.*)', output_dir)
if search is not None:
bucket_name = search.group(1)
blob_name = search.group(2)
output_folder = blob_name + '/history'
if FLAGS.is_hyperparameter_tuning:
output_folder = os.path.join(output_folder, suffix)
mu.copy_local_directory_to_gcs(history_dir, bucket_name,
output_folder)
if save_metadata:
# add meta data
dict_model['pretrained_transformer_model'] = FLAGS.pretrained_model_dir
dict_model['num_classes'] = FLAGS.num_classes
dict_data['train'] = FLAGS.input_train_tfrecords
dict_data['eval'] = FLAGS.input_eval_tfrecords
dict_parameter[
'use_decay_learning_rate'] = FLAGS.use_decay_learning_rate
dict_parameter['epochs'] = FLAGS.epochs
dict_parameter['steps_per_epoch_train'] = FLAGS.steps_per_epoch_train
dict_parameter['steps_per_epoch_eval'] = FLAGS.steps_per_epoch_eval
dict_parameter['n_steps_history'] = FLAGS.n_steps_history
dict_parameter['batch_size_train'] = FLAGS.batch_size_train
dict_parameter['batch_size_eval'] = FLAGS.batch_size_eval
dict_parameter['learning_rate'] = FLAGS.learning_rate
dict_parameter['epsilon'] = FLAGS.epsilon
dict_hardware['is_tpu'] = FLAGS.use_tpu
dict_type_job[
'is_hyperparameter_tuning'] = FLAGS.is_hyperparameter_tuning
dict_type_job['is_tpu'] = FLAGS.use_tpu
dict_software['tensorflow'] = tf.__version__
dict_software['transformer'] = __version__
dict_software['python'] = sys.version
# aggregate dictionaries
dict_all = {
'model': dict_model,
'data': dict_data,
'parameter': dict_parameter,
'hardware': dict_hardware,
'results': dict_results,
'type_job': dict_type_job,
'software': dict_software
}
# save metadata
search = re.search('gs://(.*?)/(.*)', output_dir)
if search is not None:
bucket_name = search.group(1)
blob_name = search.group(2)
output_folder = blob_name + '/metadata'
storage_client = storage.Client()
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(output_folder + '/model_job_metadata.json')
blob.upload_from_string(data=json.dumps(dict_all),
content_type='application/json')
import tensorflow as tf
from tensorboard.plugins.hparams import api as hp
# Load MNIST data using built-in datasets download function
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([16, 32]))
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd']))
METRIC_ACCURACY = 'accuracy'
with tf.summary.create_file_writer('logs/hparam_tuning').as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
)
def train_test_model(hparams):
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.relu),
tf.keras.layers.Dropout(hparams[HP_DROPOUT]),
tf.keras.layers.Dense(10, activation=tf.nn.softmax),
])
model.compile(
optimizer=hparams[HP_OPTIMIZER],
)
flags.DEFINE_integer(
"num_epochs",
5,
"Number of epochs per trial.",
)
# We'll use MNIST for this example.
DATASET = tf.keras.datasets.mnist
INPUT_SHAPE = (28, 28)
OUTPUT_CLASSES = 10
HP_CONV_LAYERS = hp.HParam("conv_layers", hp.IntInterval(1, 3))
HP_CONV_KERNEL_SIZE = hp.HParam("conv_kernel_size", hp.Discrete([3, 5]))
HP_DENSE_LAYERS = hp.HParam("dense_layers", hp.IntInterval(1, 3))
HP_DROPOUT = hp.HParam("dropout", hp.RealInterval(0.1, 0.4))
HP_OPTIMIZER = hp.HParam("optimizer", hp.Discrete(["adam", "adagrad"]))
HPARAMS = [
HP_CONV_LAYERS,
HP_CONV_KERNEL_SIZE,
HP_DENSE_LAYERS,
HP_DROPOUT,
HP_OPTIMIZER,
]
METRICS = [
hp.Metric(
"epoch_accuracy",
group="validation",
display_name="accuracy (val.)",
def random_hparam_search(cfg, data, callbacks, log_dir):
'''
Conduct a random hyperparameter search over the ranges given for the hyperparameters in config.yml and log results
in TensorBoard. Model is trained x times for y random combinations of hyperparameters.
:param cfg: Project config
:param data: Dict containing the partitioned datasets
:param callbacks: List of callbacks for Keras model (excluding TensorBoard)
:param log_dir: Base directory in which to store logs
:return: (Last model trained, resultant test set metrics, test data generator)
'''
# Define HParam objects for each hyperparameter we wish to tune.
hp_ranges = cfg['HP_SEARCH']['RANGES']
HPARAMS = []
HPARAMS.append(hp.HParam('KERNEL_SIZE', hp.Discrete(hp_ranges['KERNEL_SIZE'])))
HPARAMS.append(hp.HParam('MAXPOOL_SIZE', hp.Discrete(hp_ranges['MAXPOOL_SIZE'])))
HPARAMS.append(hp.HParam('INIT_FILTERS', hp.Discrete(hp_ranges['INIT_FILTERS'])))
HPARAMS.append(hp.HParam('FILTER_EXP_BASE', hp.IntInterval(hp_ranges['FILTER_EXP_BASE'][0], hp_ranges['FILTER_EXP_BASE'][1])))
HPARAMS.append(hp.HParam('NODES_DENSE0', hp.Discrete(hp_ranges['NODES_DENSE0'])))
HPARAMS.append(hp.HParam('CONV_BLOCKS', hp.IntInterval(hp_ranges['CONV_BLOCKS'][0], hp_ranges['CONV_BLOCKS'][1])))
HPARAMS.append(hp.HParam('DROPOUT', hp.Discrete(hp_ranges['DROPOUT'])))
HPARAMS.append(hp.HParam('LR', hp.RealInterval(hp_ranges['LR'][0], hp_ranges['LR'][1])))
HPARAMS.append(hp.HParam('OPTIMIZER', hp.Discrete(hp_ranges['OPTIMIZER'])))
HPARAMS.append(hp.HParam('L2_LAMBDA', hp.Discrete(hp_ranges['L2_LAMBDA'])))
HPARAMS.append(hp.HParam('BATCH_SIZE', hp.Discrete(hp_ranges['BATCH_SIZE'])))
HPARAMS.append(hp.HParam('IMB_STRATEGY', hp.Discrete(hp_ranges['IMB_STRATEGY'])))
# Define test set metrics that we wish to log to TensorBoard for each training run
HP_METRICS = [hp.Metric(metric, display_name='Test ' + metric) for metric in cfg['HP_SEARCH']['METRICS']]
# Configure TensorBoard to log the results
with tf.summary.create_file_writer(log_dir).as_default():
hp.hparams_config(hparams=HPARAMS, metrics=HP_METRICS)
# Complete a number of training runs at different hparam values and log the results.
repeats_per_combo = cfg['HP_SEARCH']['REPEATS'] # Number of times to train the model per combination of hparams
num_combos = cfg['HP_SEARCH']['COMBINATIONS'] # Number of random combinations of hparams to attempt
num_sessions = num_combos * repeats_per_combo # Total number of runs in this experiment
model_type = 'DCNN_BINARY' if cfg['TRAIN']['CLASS_MODE'] == 'binary' else 'DCNN_MULTICLASS'
trial_id = 0
for group_idx in range(num_combos):
rand = random.Random()
HPARAMS = {h: h.domain.sample_uniform(rand) for h in HPARAMS}
hparams = {h.name: HPARAMS[h] for h in HPARAMS} # To pass to model definition
for repeat_idx in range(repeats_per_combo):
trial_id += 1
print("Running training session %d/%d" % (trial_id, num_sessions))
print("Hparam values: ", {h.name: HPARAMS[h] for h in HPARAMS})
trial_logdir = os.path.join(log_dir, str(trial_id)) # Need specific logdir for each trial
callbacks_hp = callbacks + [TensorBoard(log_dir=trial_logdir, profile_batch=0, write_graph=False)]
# Set values of hyperparameters for this run in config file.
for h in hparams:
if h in ['LR', 'L2_LAMBDA']:
val = 10 ** hparams[h] # These hyperparameters are sampled on the log scale.
else:
val = hparams[h]
cfg['NN'][model_type][h] = val
# Set some hyperparameters that are not specified in model definition.
cfg['TRAIN']['BATCH_SIZE'] = hparams['BATCH_SIZE']
cfg['TRAIN']['IMB_STRATEGY'] = hparams['IMB_STRATEGY']
# Run a training session and log the performance metrics on the test set to HParams dashboard in TensorBoard
with tf.summary.create_file_writer(trial_logdir).as_default():
hp.hparams(HPARAMS, trial_id=str(trial_id))
model, test_metrics, test_generator = train_model(cfg, data, callbacks_hp, verbose=0)
for metric in HP_METRICS:
if metric._tag in test_metrics:
tf.summary.scalar(metric._tag, test_metrics[metric._tag], step=1) # Log test metric
return
def prueba11():
print('Prueba 11: Grid Search con un subset de los hiperparámetros')
try:
print('\n--- Lectura del dataset preprocesado')
df = pd.read_csv('../dataset/ticnn_preprocessed.csv')
except:
print('Se ha producido un error al leer el dataset')
raise
print('Dataset Leído correctamente')
try:
print('\n--- División de los datos en entrenamiento y pruebas')
# Alteración del orden
df = df.sample(frac=1.)
# Transformamos la columna text a lista de string
from ast import literal_eval
df['text'] = df['text'].apply(literal_eval)
# Aplicamos la función convert_to_string a los textos
df['text'] = df['text'].apply(model_creation.convert_to_string)
# Convertimos los textos y las targets variables a listas
texts = list(df['text'])
targets = list(df['type'])
# División en conjunto de entrenamiento y test
x_train, y_train, x_test, y_test = model_creation.train_test_split(
texts, targets)
except:
print('Error al separar en entrenamiento y pruebas')
raise
print('Datos separados correctamente')
try:
print('\n--- Creación del diccionario (Vocabulario)')
tokenizer = tf_tokenizer(num_words=20000,
oov_token='<null_token>',
lower=False,
char_level=False)
tokenizer.fit_on_texts(x_train)
except:
print('Error al crear el vocabulario')
raise
print('Vocabulario creado correctamente')
try:
print('\n--- Transformar los textos en secuencias y padding')
x_train_sequence = tokenizer.texts_to_sequences(x_train)
x_test_sequence = tokenizer.texts_to_sequences(x_test)
train_padded = pad_sequences(x_train_sequence,
maxlen=600,
dtype='int32',
truncating='post',
padding='post')
test_padded = pad_sequences(x_test_sequence,
maxlen=600,
dtype='int32',
truncating='post',
padding='post')
except:
print('Error al convertir los textos en secuencias')
raise
print('Secuenciación y padding realizados con éxito')
try:
print('\n--- Definición de los hiperparámetros a optimizar')
HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.2, 0.4))
METRIC_F1 = 'f1-score'
with tf.summary.create_file_writer(
'logs/hparam_tuning(test11)').as_default():
hp.hparams_config(
hparams=[HP_DROPOUT],
metrics=[hp.Metric(METRIC_F1, display_name='F1-Score')])
except:
print('Error al definir los hiperparámetros')
raise
print('Hiperparámetros definidos correctamente')
try:
print('\n--- Ejecución Grid Search')
session_num = 0
for dropout_rate in (HP_DROPOUT.domain.min_value,
HP_DROPOUT.domain.max_value):
hparams = {HP_DROPOUT: dropout_rate}
run_name = 'run-%d' % session_num
print('--Iniciando ejecución : %s' % run_name)
print({h: hparams[h] for h in hparams})
hparams_optimization.run('logs/hparam_tuning(test11)/' + run_name,
hparams, train_padded, test_padded,
y_train, y_test)
session_num += 1
except:
print('Error al realizar Grid Search')
raise
print(
'\n-------------------------------------------------------------------'
)
print('Grid Search realizado con éxito')
