def tuning(self): self.HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([3, 6])) self.HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2)) self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd'])) self.METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[ self.HP_NUM_UNITS, self.HP_DROPOUT, self.HP_OPTIMIZER ], metrics=[ hp.Metric(self.METRIC_ACCURACY, display_name='Accuracy') ], ) session_num = 0 for num_units in self.HP_NUM_UNITS.domain.values: for dropout_rate in (self.HP_DROPOUT.domain.min_value, self.HP_DROPOUT.domain.max_value): for optimizer in self.HP_OPTIMIZER.domain.values: hparams = { self.HP_NUM_UNITS: num_units, self.HP_DROPOUT: dropout_rate, self.HP_OPTIMIZER: optimizer, } run_name = "run-%d" % session_num print('--- Starting trial: %s' % run_name) print({h.name: hparams[h] for h in hparams}) self.run('logs/hparam_tuning/' + run_name, hparams) session_num += 1
def evaluate(self, avpool=True, query_fig=False): if self._test: test_recon_loss = 0 for x in self._test_ds: reconstructed = self._models["full"](x) test_recon_loss += np.mean( np.abs(x.numpy() - reconstructed.numpy())) self._record_scalars(test_reconstruction_loss=test_recon_loss / self._test_steps) test_ims = np.concatenate([x.numpy(), reconstructed.numpy()], 2) self._record_images(test_images=test_ims) if self._downstream_labels is not None: # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp hparams = { hp.HParam("dropout", hp.RealInterval(0., 1.)): self.config["dropout"] } else: hparams = None self._linear_classification_test(hparams, avpool=avpool, query_fig=query_fig)
def _create_hyprparameters_domain(): HP_LEARNING_RATE = hp.HParam( "learning_rate", hp.RealInterval(1.0e-4, 7.0e-3), ) HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9)) HP_LEARNING_RATE_DECAY_STEPS = hp.HParam( "learning_rate_decay_steps", hp.Discrete(range(1_000, 10_000 + 1))) HP_WEIGHT_DECAY = hp.HParam("weight_decay", hp.RealInterval(1.0e-4, 1.0e-3)) return [ HP_LEARNING_RATE, HP_BETA_1, HP_LEARNING_RATE_DECAY_STEPS, HP_WEIGHT_DECAY, ]
def _add_distributions( self, distributions: Dict[str, optuna.distributions.BaseDistribution]) -> None: for param_name, param_distribution in distributions.items(): if isinstance(param_distribution, optuna.distributions.UniformDistribution): self._hp_params[param_name] = hp.HParam( param_name, hp.RealInterval(param_distribution.low, param_distribution.high)) elif isinstance(param_distribution, optuna.distributions.LogUniformDistribution): self._hp_params[param_name] = hp.HParam( param_name, hp.RealInterval(param_distribution.low, param_distribution.high)) elif isinstance(param_distribution, optuna.distributions.DiscreteUniformDistribution): self._hp_params[param_name] = hp.HParam( param_name, hp.Discrete(param_distribution.low, param_distribution.high)) elif isinstance(param_distribution, optuna.distributions.IntUniformDistribution): self._hp_params[param_name] = hp.HParam( param_name, hp.IntInterval(param_distribution.low, param_distribution.high)) elif isinstance(param_distribution, optuna.distributions.CategoricalDistribution): self._hp_params[param_name] = hp.HParam( param_name, hp.Discrete(param_distribution.choices)) else: distribution_list = [ optuna.distributions.UniformDistribution.__name__, optuna.distributions.LogUniformDistribution.__name__, optuna.distributions.DiscreteUniformDistribution.__name__, optuna.distributions.IntUniformDistribution.__name__, optuna.distributions.CategoricalDistribution.__name__, ] raise NotImplementedError( "The distribution {} is not implemented. " "The parameter distribution should be one of the {}". format(param_distribution, distribution_list))
def _create_hyprparameters_domain(): HP_LEARNING_RATE = hp.HParam( "learning_rate", hp.RealInterval(1.0e-3, 7.0e-3), ) HP_LEARNING_RATE_DECAY_STEPS = hp.HParam( "learning_rate_decay_steps", hp.Discrete(range(1_000, 6_000 + 1)), ) HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9)) HP_FR_WEIGHT = hp.HParam("face_recognition_weight", hp.RealInterval(0.1, 1.0)) HP_SR_WEIGHT = hp.HParam("super_resolution_weight", hp.RealInterval(0.1, 1.0)) HP_PERCEPTUAL_WEIGHT = hp.HParam("perceptual_weight", hp.RealInterval(1.0e-3, 1.0e-2)) HP_GENERATOR_WEIGHT = hp.HParam("generator_weight", hp.RealInterval(1.0e-2, 7.0e-2)) HP_L1_WEIGHT = hp.HParam("l1_weight", hp.RealInterval(1.0e-2, 9.0e-2)) return [ HP_LEARNING_RATE, HP_LEARNING_RATE_DECAY_STEPS, HP_BETA_1, HP_FR_WEIGHT, HP_SR_WEIGHT, HP_PERCEPTUAL_WEIGHT, HP_GENERATOR_WEIGHT, HP_L1_WEIGHT, ]
def evaluate(self): b = tf.expand_dims(tf.expand_dims(self._buffer,0),-1) self._record_images(buffer=b) if self._downstream_labels is not None: # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp hparams = { hp.HParam("tau", hp.RealInterval(0., 10000.)):self.config["tau"], hp.HParam("alpha", hp.RealInterval(0., 1.)):self.config["alpha"], hp.HParam("batches_in_buffer", hp.IntInterval(1, 1000000)):self.config["batches_in_buffer"], hp.HParam("output_dim", hp.IntInterval(1, 1000000)):self.config["output_dim"], hp.HParam("sobel", hp.Discrete([True, False])):self.input_config["sobel"] } else: hparams=None self._linear_classification_test(hparams)
def log_hyperparameters(): """ Blueprint for hyperparameter and metric logging in tensorboard during hyperparameter tuning Returns: logparams (list): List containing the hyperparameters to log in tensorboard. metrics (list): List containing the metrics to log in tensorboard. """ logparams = [ hp.HParam( "latent_dim", hp.Discrete([2, 4, 6, 8, 12, 16]), display_name="latent_dim", description="encoding size dimensionality", ), hp.HParam( "n_components", hp.IntInterval(min_value=1, max_value=25), display_name="n_components", description="latent component number", ), hp.HParam( "gram_weight", hp.RealInterval(min_value=0.0, max_value=1.0), display_name="gram_weight", description="weight of the gram loss", ), ] metrics = [ hp.Metric( "val_number_of_populated_clusters", display_name="number of populated clusters", ), hp.Metric( "val_reconstruction_loss", display_name="reconstruction loss", ), hp.Metric( "val_gram_loss", display_name="gram loss", ), hp.Metric( "val_vq_loss", display_name="vq loss", ), hp.Metric( "val_total_loss", display_name="total loss", ), ] return logparams, metrics
def _create_hyprparameters_domain(): HP_LEARNING_RATE = hp.HParam( "learning_rate", hp.RealInterval(1.0e-3, 7.0e-3), ) HP_BETA_1 = hp.HParam("beta_1", hp.RealInterval(0.5, 0.9)) HP_PERCEPTUAL_WEIGHT = hp.HParam("perceptual_weight", hp.RealInterval(1.0e-3, 1.0)) HP_GENERATOR_WEIGHT = hp.HParam("generator_weight", hp.RealInterval(1.0e-3, 7.0e-2)) HP_L1_WEIGHT = hp.HParam("l1_weight", hp.RealInterval(1.0e-2, 9.0e-2)) return [ HP_LEARNING_RATE, HP_BETA_1, HP_PERCEPTUAL_WEIGHT, HP_GENERATOR_WEIGHT, HP_L1_WEIGHT, ]
def __init__(self, lr_range=(1.e-8, 1.), momentum_range=(0., 1.)): super(SGDMomentumManager, self).__init__(lr_range=lr_range) self._hparam['momentum'] = hp.HParam( 'momentum', description="Momentum Rate", domain=hp.RealInterval(*momentum_range)) self._hparam['nesterov'] = hp.HParam('nesterov', description="Momentum", domain=hp.Discrete( ['standard', 'nesterov']))
def convert_hyperparams_to_hparams( hyperparams: hp_module.HyperParameters ) -> Dict[hparams_api.HParam, Any]: """Converts KerasTuner HyperParameters to TensorBoard HParams. Args: hyperparams: A KerasTuner HyperParameters instance Returns: A dict that maps TensorBoard HParams to current values. """ hparams = {} for hp in hyperparams.space: hparams_value = {} try: hparams_value = hyperparams.get(hp.name) except ValueError: continue hparams_domain = {} if isinstance(hp, hp_module.Choice): hparams_domain = hparams_api.Discrete(hp.values) elif isinstance(hp, hp_module.Int): if hp.step is None or hp.step == 1: hparams_domain = hparams_api.IntInterval( hp.min_value, hp.max_value) else: # Note: `hp.max_value` is inclusive, unlike the end index # of Python `range()`, which is exclusive values = list(range(hp.min_value, hp.max_value + 1, hp.step)) hparams_domain = hparams_api.Discrete(values) elif isinstance(hp, hp_module.Float): if hp.step is None: hparams_domain = hparams_api.RealInterval( hp.min_value, hp.max_value) else: # Note: `hp.max_value` is inclusive, which is also # the default for Numpy's linspace num_samples = int((hp.max_value - hp.min_value) / hp.step) end_value = hp.min_value + (num_samples * hp.step) values = np.linspace(hp.min_value, end_value, num_samples + 1).tolist() hparams_domain = hparams_api.Discrete(values) elif isinstance(hp, hp_module.Boolean): hparams_domain = hparams_api.Discrete([True, False]) elif isinstance(hp, hp_module.Fixed): hparams_domain = hparams_api.Discrete([hp.value]) else: raise ValueError( "`HyperParameter` type not recognized: {}".format(hp)) hparams_key = hparams_api.HParam(hp.name, hparams_domain) hparams[hparams_key] = hparams_value return hparams
def evaluate(self, avpool=True): predictions = self._models["full"].predict(self._val_ds) num_categories = len(self.categories) for i in range(num_categories): category = self.categories[i] preds = predictions[:, i] y_true = self._val_labels[:, i] acc = np.mean(y_true == (preds >= 0.5).astype(int)) auc = roc_auc_score(y_true, preds) self._record_scalars(**{ f"val_accuracy_{category}": acc, f"val_auc_{category}": auc }, metric=True) # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp metrics = [] for c in self.categories: metrics.append(f"val_accuracy_{c}") metrics.append(f"val_auc_{c}") hparams = { hp.HParam("lam", hp.RealInterval(0., 10000.)): self.config["lam"], hp.HParam("tau", hp.RealInterval(0., 10000.)): self.config["tau"], hp.HParam("mu", hp.RealInterval(0., 10000.)): self.config["mu"], hp.HParam("batch_size", hp.RealInterval(0., 10000.)): self.input_config["batch_size"], hp.HParam("lr", hp.RealInterval(0., 10000.)): self.config["lr"], hp.HParam("lr_decay", hp.RealInterval(0., 10000.)): self.config["lr_decay"], hp.HParam("decay_type", hp.Discrete(["cosine", "exponential", "staircase"])): self.config["decay_type"], hp.HParam("opt_type", hp.Discrete(["sgd", "adam", "momentum"])): self.config["opt_type"], hp.HParam("weight_decay", hp.RealInterval(0., 10000.)): self.config["weight_decay"] } self._hparams_config = hp.hparams_config( hparams=list(hparams.keys()), metrics=[hp.Metric(m) for m in metrics]) # record hyperparameters base_dir, run_name = os.path.split(self.logdir) if len(run_name) == 0: base_dir, run_name = os.path.split(base_dir)
def __init__(self, parent=None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.onBindingUI() self.batch_size = 32 self.learning_rate = 0.001 self.optimizer = 'sgd' self.HP_BATCH_SIZE = hp.HParam('num_units', hp.Discrete([self.batch_size, self.batch_size])) self.HP_LEARNING_RATE = hp.HParam('dropout', hp.RealInterval(self.learning_rate, self.learning_rate)) self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete([self.optimizer]))
def __init__(self, lr_range=(1.e-8, 1.)): """Create a SGD manager with a specific learning rate range""" super(SGDManager, self).__init__() self._hparam = { "optimizer": hp.HParam("optimizer", domain=hp.Discrete(['SGD']), display_name="Optimizer"), "learning_rate": hp.HParam("learning_rate", domain=hp.RealInterval(*lr_range), display_name="Learning rate"), }
def evaluate(self): num_test_images = 10 if self._test: preds = self._models["inpainter"].predict(self._test_masked_ims) preds = preds[:, :self.input_config["imshape"][0], :self. input_config["imshape"][1], :] reconstruction_residual = self._test_mask * (preds - self._test_ims) reconstructed_loss = np.mean(np.abs(reconstruction_residual)) # see how the discriminator does on them disc_outputs_on_raw = self._models["discriminator"].predict( self._test_ims) disc_outputs_on_inpaint = self._models["discriminator"].predict( preds) # for the visualization in tensorboard: replace the unmasked areas # with the input image as a guide to the eye preds = preds * self._test_mask + self._test_ims * ( 1 - self._test_mask) predviz = np.concatenate([ self._test_masked_ims[:num_test_images], preds[:num_test_images] ], 2).astype(np.float32) # record all the summaries tf.summary.image("inpaints", predviz, step=self.step, max_outputs=10) tf.summary.histogram("disc_outputs_on_raw", disc_outputs_on_raw, step=self.step) tf.summary.histogram("disc_outputs_on_inpaint", disc_outputs_on_inpaint, step=self.step) self._record_scalars(test_recon_loss=reconstructed_loss) if self._downstream_labels is not None: # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp hparams = { hp.HParam("adv_weight", hp.RealInterval(0., 10000.)): self.config["adv_weight"], hp.HParam("sobel", hp.Discrete([True, False])): self.input_config["sobel"] } else: hparams = None self._linear_classification_test(hparams)
def kt_to_hparam(hp: kt.HyperParameter) -> hp_lib.HParam: if isinstance(hp, kt.engine.hyperparameters.Float): domain = hp_lib.RealInterval(hp.min_value, hp.max_value) elif isinstance(hp, kt.engine.hyperparameters.Int): domain = hp_lib.IntInterval(hp.min_value, hp.max_value) elif isinstance(hp, kt.engine.hyperparameters.Boolean): domain = hp_lib.Discrete([False, True], dtype=bool) elif isinstance(hp, kt.engine.hyperparameters.Fixed): domain = hp_lib.Discrete([hp.value]) elif isinstance(hp, kt.engine.hyperparameters.Choice): domain = hp_lib.Discrete(hp.values) else: raise TypeError(f"Unsupposed hyperparamter type {hp}") return hp_lib.HParam(hp.name, domain)
def __init__(self, lr_range=(1.e-8, 1.), beta_1_range=(1. - 1.e-1, 1. - 1.e-4), beta_2_range=(1. - 1.e-3, 1. - 1e-6), epsilon_range=(1.e-8, 1.e-3)): """Create an Adam/Adamax manager with a specific range for hyperparameters A last non-tunable hyperparameter is `optimizer`, which can be Adam, Amsgrad, Adamax. Args: lr_range (): beta_1_range (): beta_2_range (): epsilon_range (): """ super(AdamManager, self).__init__() self._hparam = { "optimizer": hp.HParam("optimizer", domain=hp.Discrete(['Adam', 'Amsgrad', 'Adamax']), display_name="Optimizer"), "learning_rate": hp.HParam("learning_rate", domain=hp.RealInterval(*lr_range), display_name="Learning rate"), "beta_1": hp.HParam("beta_1", domain=hp.RealInterval(*beta_1_range), display_name="Beta1"), "beta_2": hp.HParam("beta_2", domain=hp.RealInterval(*beta_2_range), display_name="Beta2"), "epsilon": hp.HParam("epsilon", domain=hp.RealInterval(*epsilon_range), display_name="Epsilon"), }
def evaluate(self): if self._downstream_labels is not None: # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp hparams = { hp.HParam("temperature", hp.RealInterval(0., 10000.)): self.config["temperature"], hp.HParam("num_hidden", hp.IntInterval(1, 1000000)): self.config["num_hidden"], hp.HParam("output_dim", hp.IntInterval(1, 1000000)): self.config["output_dim"] } else: hparams = None self._linear_classification_test(hparams)
def prepare_hparams(self, hparams_domains): """Convert informal specifications to a dict of hp.HParam""" domains = {} domains["num_layers"] = hp.HParam( "num_layers", hp.Discrete(hparams_domains["num_layers"])) domains["learning_rate"] = hp.HParam( "learning_rate", hp.RealInterval(*hparams_domains["learning_rate"])) domains["num_filters"] = hp.HParam( "num_filters", hp.Discrete(hparams_domains["num_filters"])) domains["batch_normalization"] = hp.HParam( "batch_normalization", hp.Discrete(hparams_domains["batch_normalization"])) return domains
def _add_distributions( self, distributions: Dict[str, optuna.distributions.BaseDistribution]) -> None: real_distributions = ( optuna.distributions.UniformDistribution, optuna.distributions.LogUniformDistribution, optuna.distributions.DiscreteUniformDistribution, optuna.distributions.FloatDistribution, ) int_distributions = ( optuna.distributions.IntUniformDistribution, optuna.distributions.IntLogUniformDistribution, optuna.distributions.IntDistribution, ) categorical_distributions = ( optuna.distributions.CategoricalDistribution, ) supported_distributions = (real_distributions + int_distributions + categorical_distributions) for param_name, param_distribution in distributions.items(): if isinstance(param_distribution, real_distributions): self._hp_params[param_name] = hp.HParam( param_name, hp.RealInterval(float(param_distribution.low), float(param_distribution.high)), ) elif isinstance(param_distribution, int_distributions): self._hp_params[param_name] = hp.HParam( param_name, hp.IntInterval(param_distribution.low, param_distribution.high), ) elif isinstance(param_distribution, categorical_distributions): self._hp_params[param_name] = hp.HParam( param_name, hp.Discrete(param_distribution.choices), ) else: distribution_list = [ distribution.__name__ for distribution in supported_distributions ] raise NotImplementedError( "The distribution {} is not implemented. " "The parameter distribution should be one of the {}". format(param_distribution, distribution_list))
def convert_hyperparams_to_hparams(hyperparams): """Converts KerasTuner HyperParameters to TensorBoard HParams.""" hparams = {} for hp in hyperparams.space: hparams_value = {} try: hparams_value = hyperparams.get(hp.name) except ValueError: continue hparams_domain = {} if isinstance(hp, hp_module.Choice): hparams_domain = hparams_api.Discrete(hp.values) elif isinstance(hp, hp_module.Int): if hp.step is not None and hp.step != 1: # Note: `hp.max_value` is inclusive, unlike the end index # of Python `range()`, which is exclusive values = list(range(hp.min_value, hp.max_value + 1, hp.step)) hparams_domain = hparams_api.Discrete(values) else: hparams_domain = hparams_api.IntInterval( hp.min_value, hp.max_value) elif isinstance(hp, hp_module.Float): if hp.step is not None: # Note: `hp.max_value` is inclusive, unlike the end index # of Numpy's arange(), which is exclusive values = np.arange(hp.min_value, hp.max_value + 1e-7, step=hp.step).tolist() hparams_domain = hparams_api.Discrete(values) else: hparams_domain = hparams_api.RealInterval( hp.min_value, hp.max_value) elif isinstance(hp, hp_module.Boolean): hparams_domain = hparams_api.Discrete([True, False]) elif isinstance(hp, hp_module.Fixed): hparams_domain = hparams_api.Discrete([hp.value]) else: raise ValueError( "`HyperParameter` type not recognized: {}".format(hp)) hparams_key = hparams_api.HParam(hp.name, hparams_domain) hparams[hparams_key] = hparams_value return hparams
def __init__(self, num_session_groups=10, HP_CONV_LAYERS=hp.HParam("conv_layers", hp.IntInterval(1, 3)), HP_CONV_KERNEL_SIZE=hp.HParam("conv_kernel_size", hp.Discrete([3, 5])), HP_DENSE_LAYERS=hp.HParam("dense_layers", hp.IntInterval(1, 3)), HP_DROPOUT=hp.HParam("dropout", hp.RealInterval(0.1, 0.4)), HP_OPTIMIZER=hp.HParam("optimizer", hp.Discrete(["adam", "adagrad"]))): self.HP_CONV_LAYERS = HP_CONV_LAYERS self.HP_CONV_KERNEL_SIZE = HP_CONV_KERNEL_SIZE self.HP_DENSE_LAYERS = HP_DENSE_LAYERS self.HP_DROPOUT = HP_DROPOUT self.HP_OPTIMIZER = HP_OPTIMIZER self.num_session_groups = num_session_groups self.HPARAMS = [ HP_CONV_LAYERS, HP_CONV_KERNEL_SIZE, HP_DENSE_LAYERS, HP_DROPOUT, HP_OPTIMIZER, ] self.METRICS = [ hp.Metric( "epoch_accuracy", group="train", display_name="accuracy (train)", ), hp.Metric( "epoch_loss", group="train", display_name="loss (train)", ), hp.Metric( "epoch_accuracy", group="validation", display_name="accuracy (val.)", ), hp.Metric( "epoch_loss", group="validation", display_name="loss (val.)", ) ]
def hparam_from_config_space_item(self, c_item): name = c_item['config']['name'] p_type = c_item['class_name'] cfg = c_item['config'] # parm = hp.Discrete() hparam = None if p_type == 'Int': hparam = hp.HParam(name=name, domain=hp.IntInterval(cfg['min_value'], cfg['max_value'])) elif p_type == 'Float': hparam = hp.HParam(name=name, domain=hp.RealInterval(cfg['min_value'], cfg['max_value'])) elif p_type == 'Fixed': hparam = hp.HParam(name=name, domain=hp.Discrete([cfg['value']])) elif p_type == 'Choice': hparam = hp.HParam(name=name, domain=hp.Discrete(cfg['values'])) return hparam
def evaluate(self): if self._test: # compute features- in memory for now f_x = [] f_y = [] for x, y in self._test_ds: f_x.append(self.flatten(self.fcn(x))) f_y.append(self.flatten(self.fcn(y))) f_x = tf.concat(f_x, 0) f_y = tf.concat(f_y, 0) # compute P for each head for e, h in enumerate(self.heads): P = compute_p(f_x, f_y, h) self._record_images(**{"P_head_%s"%e:P}) for e, h in enumerate(self.heads_oc): P = compute_p(f_x, f_y, h) self._record_images(**{"P_oc_head_%s"%e:P}) if self._downstream_labels is not None: # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp hparams = { hp.HParam("k", hp.IntInterval(0, 1000000), description="output dimension for clustering head"):self.config["k"], hp.HParam("h", hp.IntInterval(0, 1000000), description="number of clustering sub-heads"):self.config["h"], hp.HParam("k_oc", hp.IntInterval(0, 1000000), description="output dimension for overclustering head"):self.config["k_oc"], hp.HParam("entropy_weight", hp.RealInterval(0., 1000000.), description="additional weight factor for entropy in loss function"):self.config["entropy_weight"], hp.HParam("r", hp.IntInterval(0, 1000000), description="number of times to repeat each image sequentially in the data pipeline"):self.config["r"], hp.HParam("sobel", hp.Discrete([True, False]), description="whether Sobel filtering was applied to inputs"):self.input_config["sobel"] } else: hparams=None self._linear_classification_test(hparams)
def __init__(self, *, input_size, n_classes, activations=("relu", ), losses=("categorical_crossentropy", )): self.input_size = input_size self.n_classes = n_classes self._hparam = { "n_layers": hp.HParam("n_layers", domain=hp.IntInterval(1, 100), display_name="Depth"), "layer_size": hp.HParam("layer_size", domain=hp.IntInterval(1, 100), display_name="Width"), "layer_activation": hp.HParam("layer_activation", domain=hp.Discrete(activations), display_name="Activation Fct."), "reg": hp.HParam("reg", domain=hp.RealInterval(0., 1.), display_name="Reg."), "loss_function": hp.HParam("loss_function", domain=hp.Discrete(losses), display_name="Loss Fct.") } self._metrics = { "accuracy": hp.Metric("accuracy", display_name="Accuracy") } self._model = None
def _configure_hparams(logdir, dicts, metrics=["linear_classification_accuracy", "alignment", "uniformity"]): """ Set up the tensorboard hyperparameter interface :logdir: string; path to log directory :dicts: list of dictionaries containing hyperparameter values :metrics: list of strings; metric names """ metrics = [hp.Metric(m) for m in metrics] params = {} # for each parameter dictionary for d in dicts: # for each parameter: for k in d: # is it a categorical? if k in SPECIAL_HPARAMS: params[hp.HParam(k, hp.Discrete(SPECIAL_HPARAMS[k]))] = d[k] elif isinstance(d[k], bool): params[hp.HParam(k, hp.Discrete([True, False]))] = d[k] elif isinstance(d[k], int): params[hp.HParam(k, hp.IntInterval(1, 1000000))] = d[k] elif isinstance(d[k], float): params[hp.HParam(k, hp.RealInterval(0., 10000000.))] = d[k] # hparams_config = hp.hparams_config( hparams=list(params.keys()), metrics=metrics) # get a name for the run base_dir, run_name = os.path.split(logdir) if len(run_name) == 0: base_dir, run_name = os.path.split(base_dir) # record hyperparamers hp.hparams(params, trial_id=run_name)
def train_and_evaluate( model, num_epochs, steps_per_epoch, train_data, validation_steps, eval_data, output_dir, n_steps_history, FLAGS, decay_type, learning_rate=3e-5, s=1, n_batch_decay=1, metric_accuracy='metric', ): """ Compiles keras model and loads data into it for training. """ logging.info('training the model ...') model_callbacks = [] # create meta data dictionary dict_model = {} dict_data = {} dict_parameter = {} dict_hardware = {} dict_results = {} dict_type_job = {} dict_software = {} # for debugging only activate_tensorboard = True activate_hp_tensorboard = False # True activate_lr = False save_checkpoints = False # True save_history_per_step = False # True save_metadata = False # True activate_timing = False # True # drop official method that is not working activate_tf_summary_hp = True # False # hardcoded way of doing hp activate_hardcoded_hp = True # True # dependencies if activate_tf_summary_hp: save_history_per_step = True if FLAGS.is_hyperparameter_tuning: # get trial ID suffix = mu.get_trial_id() if suffix == '': logging.error('No trial ID for hyper parameter job!') FLAGS.is_hyperparameter_tuning = False else: # callback for hp logging.info('Creating a callback to store the metric!') if activate_tf_summary_hp: hp_metric = mu.HP_metric(metric_accuracy) model_callbacks.append(hp_metric) if output_dir: if activate_tensorboard: # tensorflow callback log_dir = os.path.join(output_dir, 'tensorboard') if FLAGS.is_hyperparameter_tuning: log_dir = os.path.join(log_dir, suffix) tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=log_dir, histogram_freq=1, embeddings_freq=0, write_graph=True, update_freq='batch', profile_batch='10, 20') model_callbacks.append(tensorboard_callback) if save_checkpoints: # checkpoints callback checkpoint_dir = os.path.join(output_dir, 'checkpoint_model') if not FLAGS.is_hyperparameter_tuning: # not saving model during hyper parameter tuning # heckpoint_dir = os.path.join(checkpoint_dir, suffix) checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt_{epoch:02d}') checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_prefix, verbose=1, save_weights_only=True) model_callbacks.append(checkpoint_callback) if activate_lr: # decay learning rate callback # code snippet to make the switching between different learning rate decays possible if decay_type == 'exponential': decay_fn = mu.exponential_decay(lr0=learning_rate, s=s) elif decay_type == 'stepwise': decay_fn = mu.step_decay(lr0=learning_rate, s=s) elif decay_type == 'timebased': decay_fn = mu.time_decay(lr0=learning_rate, s=s) else: decay_fn = mu.no_decay(lr0=learning_rate) # exponential_decay_fn = mu.exponential_decay(lr0=learning_rate, s=s) # lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn, verbose=1) # model_callbacks.append(lr_scheduler) # added these two lines for batch updates lr_decay_batch = mu.LearningRateSchedulerPerBatch(decay_fn, n_batch_decay, verbose=1) # lr_decay_batch = mu.LearningRateSchedulerPerBatch(exponential_decay_fn, n_batch_decay, verbose=0) # lambda step: ((learning_rate - min_learning_rate) * decay_rate ** step + min_learning_rate)) model_callbacks.append(lr_decay_batch) # print_lr = mu.PrintLR() # model_callbacks.append(mu.PrintLR()) # --------------------------------------------------------------------------------------------------------------- # callback to store all the learning rates # all_learning_rates = mu.LearningRateSchedulerPerBatch(model.optimizer, n_steps_history) # all_learning_rates = mu.LR_per_step() # all_learning_rates = mu.LR_per_step(model.optimizer) # model_callbacks.append(all_learning_rates) # disble if save_history_per_step: # callback to create history per step (not per epoch) histories_per_step = mu.History_per_step(eval_data, n_steps_history) model_callbacks.append(histories_per_step) if activate_timing: # callback to time each epoch timing = mu.TimingCallback() model_callbacks.append(timing) # checking model callbacks for logging.info('model\'s callback:\n {}'.format(str(model_callbacks))) # train the model # time the function start_time = time.time() logging.info('starting model.fit') # verbose = 0 (silent) # verbose = 1 (progress bar) # verbose = 2 (one line per epoch) verbose = 1 history = model.fit(train_data, epochs=num_epochs, steps_per_epoch=steps_per_epoch, validation_data=eval_data, validation_steps=validation_steps, verbose=verbose, callbacks=model_callbacks) # print execution time elapsed_time_secs = time.time() - start_time logging.info('\nexecution time: {}'.format( timedelta(seconds=round(elapsed_time_secs)))) # check model logging.info('model summary ={}'.format(model.summary())) logging.info('model input ={}'.format(model.inputs)) logging.info('model outputs ={}'.format(model.outputs)) # to be remove logging.info('\ndebugging .... : ') pp.print_info_data(train_data) if activate_timing: logging.info('timing per epoch:\n{}'.format( list( map(lambda x: str(timedelta(seconds=round(x))), timing.timing_epoch)))) logging.info('timing per validation:\n{}'.format( list( map(lambda x: str(timedelta(seconds=round(x))), timing.timing_valid)))) logging.info('sum timing over all epochs:\n{}'.format( timedelta(seconds=round(sum(timing.timing_epoch))))) # for hp parameter tuning in TensorBoard if FLAGS.is_hyperparameter_tuning: logging.info('setup hyperparameter tuning!') # test #params = json.loads(os.environ.get("CLUSTER_SPEC", "{}")).get("job", {}) #print('debug: CLUSTER_SPEC1:', params) #params = json.loads(os.environ.get("CLUSTER_SPEC", "{}")).get("job", {}).get("job_args", {}) #print('debug: CLUSTER_SPEC2:', params) logging.info('debug: os.environ.items():', os.environ.items()) # if activate_hardcoded_hp: # trick to bypass ai platform bug logging.info('hardcoded hyperparameter tuning!') value_accuracy = histories_per_step.accuracies[-1] hpt = hypertune.HyperTune() hpt.report_hyperparameter_tuning_metric( hyperparameter_metric_tag=metric_accuracy, metric_value=value_accuracy, global_step=0) else: # should be extracted from /var/hypertune/output.metric logging.info('standard hyperparameter tuning!') # is this needed ? # value_accuracy = histories_per_step.accuracies[-1] # look at the content of the file path_metric = '/var/hypertune/output.metric' logging.info('checking if /var/hypertune/output.metric exist!') if os.path.isfile(path_metric): logging.info('file {} exist !'.format(path_metric)) with open(path_metric, 'r') as f: logging.info('content of output.metric: {}'.format(f.read())) if activate_hp_tensorboard: logging.info('setup TensorBoard for hyperparameter tuning!') # CAIP #params = json.loads(os.environ.get("TF_CONFIG", "{}")).get("job", {}).get("hyperparameters", {}).get("params", {}) #uCAIP params = json.loads( os.environ.get("CLUSTER_SPEC", "{}") ) #.get("job", {}).get("hyperparameters", {}).get("params", {}) print('debug: CLUSTER_SPEC:', params) list_hp = [] hparams = {} for el in params: hp_dict = dict(el) if hp_dict.get('type') == 'DOUBLE': key_hp = hp.HParam( hp_dict.get('parameter_name'), hp.RealInterval(hp_dict.get('min_value'), hp_dict.get('max_value'))) list_hp.append(key_hp) try: hparams[key_hp] = FLAGS[hp_dict.get( 'parameter_name')].value except KeyError: logging.error( 'hyperparameter key {} doesn\'t exist'.format( hp_dict.get('parameter_name'))) hparams_dir = os.path.join(output_dir, 'hparams_tuning') with tf.summary.create_file_writer(hparams_dir).as_default(): hp.hparams_config( hparams=list_hp, metrics=[ hp.Metric(metric_accuracy, display_name=metric_accuracy) ], ) hparams_dir = os.path.join(hparams_dir, suffix) with tf.summary.create_file_writer(hparams_dir).as_default(): # record the values used in this trial hp.hparams(hparams) tf.summary.scalar(metric_accuracy, value_accuracy, step=1) if save_history_per_step: # save the history in a file search = re.search('gs://(.*?)/(.*)', output_dir) if search is not None: # temp folder locally and to be ove on gcp later history_dir = os.path.join('./', model.name) os.makedirs(history_dir, exist_ok=True) else: # locally history_dir = os.path.join(output_dir, model.name) os.makedirs(history_dir, exist_ok=True) logging.debug('history_dir: \n {}'.format(history_dir)) with open(history_dir + '/history', 'wb') as file: model_history = mu.History_trained_model(history.history, history.epoch, history.params) pickle.dump(model_history, file, pickle.HIGHEST_PROTOCOL) with open(history_dir + '/history_per_step', 'wb') as file: model_history_per_step = mu.History_per_steps_trained_model( histories_per_step.steps, histories_per_step.losses, histories_per_step.accuracies, histories_per_step.val_steps, histories_per_step.val_losses, histories_per_step.val_accuracies, 0, # all_learning_rates.all_lr, 0, # all_learning_rates.all_lr_alternative, 0) # all_learning_rates.all_lr_logs) pickle.dump(model_history_per_step, file, pickle.HIGHEST_PROTOCOL) if output_dir: # save the model savemodel_path = os.path.join(output_dir, 'saved_model') if not FLAGS.is_hyperparameter_tuning: # not saving model during hyper parameter tuning # savemodel_path = os.path.join(savemodel_path, suffix) model.save(os.path.join(savemodel_path, model.name)) model2 = tf.keras.models.load_model( os.path.join(savemodel_path, model.name)) # check model logging.info('model2 summary ={}'.format(model2.summary())) logging.info('model2 input ={}'.format(model2.inputs)) logging.info('model2 outputs ={}'.format(model2.outputs)) logging.info('model2 signature outputs ={}'.format( model2.signatures['serving_default'].structured_outputs)) logging.info('model2 inputs ={}'.format( model2.signatures['serving_default'].inputs[0])) if save_history_per_step: # save history search = re.search('gs://(.*?)/(.*)', output_dir) if search is not None: bucket_name = search.group(1) blob_name = search.group(2) output_folder = blob_name + '/history' if FLAGS.is_hyperparameter_tuning: output_folder = os.path.join(output_folder, suffix) mu.copy_local_directory_to_gcs(history_dir, bucket_name, output_folder) if save_metadata: # add meta data dict_model['pretrained_transformer_model'] = FLAGS.pretrained_model_dir dict_model['num_classes'] = FLAGS.num_classes dict_data['train'] = FLAGS.input_train_tfrecords dict_data['eval'] = FLAGS.input_eval_tfrecords dict_parameter[ 'use_decay_learning_rate'] = FLAGS.use_decay_learning_rate dict_parameter['epochs'] = FLAGS.epochs dict_parameter['steps_per_epoch_train'] = FLAGS.steps_per_epoch_train dict_parameter['steps_per_epoch_eval'] = FLAGS.steps_per_epoch_eval dict_parameter['n_steps_history'] = FLAGS.n_steps_history dict_parameter['batch_size_train'] = FLAGS.batch_size_train dict_parameter['batch_size_eval'] = FLAGS.batch_size_eval dict_parameter['learning_rate'] = FLAGS.learning_rate dict_parameter['epsilon'] = FLAGS.epsilon dict_hardware['is_tpu'] = FLAGS.use_tpu dict_type_job[ 'is_hyperparameter_tuning'] = FLAGS.is_hyperparameter_tuning dict_type_job['is_tpu'] = FLAGS.use_tpu dict_software['tensorflow'] = tf.__version__ dict_software['transformer'] = __version__ dict_software['python'] = sys.version # aggregate dictionaries dict_all = { 'model': dict_model, 'data': dict_data, 'parameter': dict_parameter, 'hardware': dict_hardware, 'results': dict_results, 'type_job': dict_type_job, 'software': dict_software } # save metadata search = re.search('gs://(.*?)/(.*)', output_dir) if search is not None: bucket_name = search.group(1) blob_name = search.group(2) output_folder = blob_name + '/metadata' storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) blob = bucket.blob(output_folder + '/model_job_metadata.json') blob.upload_from_string(data=json.dumps(dict_all), content_type='application/json')
import tensorflow as tf from tensorboard.plugins.hparams import api as hp # Load MNIST data using built-in datasets download function mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([16, 32])) HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2)) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd'])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) def train_test_model(hparams): model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(), tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.relu), tf.keras.layers.Dropout(hparams[HP_DROPOUT]), tf.keras.layers.Dense(10, activation=tf.nn.softmax), ]) model.compile( optimizer=hparams[HP_OPTIMIZER],
) flags.DEFINE_integer( "num_epochs", 5, "Number of epochs per trial.", ) # We'll use MNIST for this example. DATASET = tf.keras.datasets.mnist INPUT_SHAPE = (28, 28) OUTPUT_CLASSES = 10 HP_CONV_LAYERS = hp.HParam("conv_layers", hp.IntInterval(1, 3)) HP_CONV_KERNEL_SIZE = hp.HParam("conv_kernel_size", hp.Discrete([3, 5])) HP_DENSE_LAYERS = hp.HParam("dense_layers", hp.IntInterval(1, 3)) HP_DROPOUT = hp.HParam("dropout", hp.RealInterval(0.1, 0.4)) HP_OPTIMIZER = hp.HParam("optimizer", hp.Discrete(["adam", "adagrad"])) HPARAMS = [ HP_CONV_LAYERS, HP_CONV_KERNEL_SIZE, HP_DENSE_LAYERS, HP_DROPOUT, HP_OPTIMIZER, ] METRICS = [ hp.Metric( "epoch_accuracy", group="validation", display_name="accuracy (val.)",
def random_hparam_search(cfg, data, callbacks, log_dir): ''' Conduct a random hyperparameter search over the ranges given for the hyperparameters in config.yml and log results in TensorBoard. Model is trained x times for y random combinations of hyperparameters. :param cfg: Project config :param data: Dict containing the partitioned datasets :param callbacks: List of callbacks for Keras model (excluding TensorBoard) :param log_dir: Base directory in which to store logs :return: (Last model trained, resultant test set metrics, test data generator) ''' # Define HParam objects for each hyperparameter we wish to tune. hp_ranges = cfg['HP_SEARCH']['RANGES'] HPARAMS = [] HPARAMS.append(hp.HParam('KERNEL_SIZE', hp.Discrete(hp_ranges['KERNEL_SIZE']))) HPARAMS.append(hp.HParam('MAXPOOL_SIZE', hp.Discrete(hp_ranges['MAXPOOL_SIZE']))) HPARAMS.append(hp.HParam('INIT_FILTERS', hp.Discrete(hp_ranges['INIT_FILTERS']))) HPARAMS.append(hp.HParam('FILTER_EXP_BASE', hp.IntInterval(hp_ranges['FILTER_EXP_BASE'][0], hp_ranges['FILTER_EXP_BASE'][1]))) HPARAMS.append(hp.HParam('NODES_DENSE0', hp.Discrete(hp_ranges['NODES_DENSE0']))) HPARAMS.append(hp.HParam('CONV_BLOCKS', hp.IntInterval(hp_ranges['CONV_BLOCKS'][0], hp_ranges['CONV_BLOCKS'][1]))) HPARAMS.append(hp.HParam('DROPOUT', hp.Discrete(hp_ranges['DROPOUT']))) HPARAMS.append(hp.HParam('LR', hp.RealInterval(hp_ranges['LR'][0], hp_ranges['LR'][1]))) HPARAMS.append(hp.HParam('OPTIMIZER', hp.Discrete(hp_ranges['OPTIMIZER']))) HPARAMS.append(hp.HParam('L2_LAMBDA', hp.Discrete(hp_ranges['L2_LAMBDA']))) HPARAMS.append(hp.HParam('BATCH_SIZE', hp.Discrete(hp_ranges['BATCH_SIZE']))) HPARAMS.append(hp.HParam('IMB_STRATEGY', hp.Discrete(hp_ranges['IMB_STRATEGY']))) # Define test set metrics that we wish to log to TensorBoard for each training run HP_METRICS = [hp.Metric(metric, display_name='Test ' + metric) for metric in cfg['HP_SEARCH']['METRICS']] # Configure TensorBoard to log the results with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config(hparams=HPARAMS, metrics=HP_METRICS) # Complete a number of training runs at different hparam values and log the results. repeats_per_combo = cfg['HP_SEARCH']['REPEATS'] # Number of times to train the model per combination of hparams num_combos = cfg['HP_SEARCH']['COMBINATIONS'] # Number of random combinations of hparams to attempt num_sessions = num_combos * repeats_per_combo # Total number of runs in this experiment model_type = 'DCNN_BINARY' if cfg['TRAIN']['CLASS_MODE'] == 'binary' else 'DCNN_MULTICLASS' trial_id = 0 for group_idx in range(num_combos): rand = random.Random() HPARAMS = {h: h.domain.sample_uniform(rand) for h in HPARAMS} hparams = {h.name: HPARAMS[h] for h in HPARAMS} # To pass to model definition for repeat_idx in range(repeats_per_combo): trial_id += 1 print("Running training session %d/%d" % (trial_id, num_sessions)) print("Hparam values: ", {h.name: HPARAMS[h] for h in HPARAMS}) trial_logdir = os.path.join(log_dir, str(trial_id)) # Need specific logdir for each trial callbacks_hp = callbacks + [TensorBoard(log_dir=trial_logdir, profile_batch=0, write_graph=False)] # Set values of hyperparameters for this run in config file. for h in hparams: if h in ['LR', 'L2_LAMBDA']: val = 10 ** hparams[h] # These hyperparameters are sampled on the log scale. else: val = hparams[h] cfg['NN'][model_type][h] = val # Set some hyperparameters that are not specified in model definition. cfg['TRAIN']['BATCH_SIZE'] = hparams['BATCH_SIZE'] cfg['TRAIN']['IMB_STRATEGY'] = hparams['IMB_STRATEGY'] # Run a training session and log the performance metrics on the test set to HParams dashboard in TensorBoard with tf.summary.create_file_writer(trial_logdir).as_default(): hp.hparams(HPARAMS, trial_id=str(trial_id)) model, test_metrics, test_generator = train_model(cfg, data, callbacks_hp, verbose=0) for metric in HP_METRICS: if metric._tag in test_metrics: tf.summary.scalar(metric._tag, test_metrics[metric._tag], step=1) # Log test metric return
def prueba11(): print('Prueba 11: Grid Search con un subset de los hiperparámetros') try: print('\n--- Lectura del dataset preprocesado') df = pd.read_csv('../dataset/ticnn_preprocessed.csv') except: print('Se ha producido un error al leer el dataset') raise print('Dataset Leído correctamente') try: print('\n--- División de los datos en entrenamiento y pruebas') # Alteración del orden df = df.sample(frac=1.) # Transformamos la columna text a lista de string from ast import literal_eval df['text'] = df['text'].apply(literal_eval) # Aplicamos la función convert_to_string a los textos df['text'] = df['text'].apply(model_creation.convert_to_string) # Convertimos los textos y las targets variables a listas texts = list(df['text']) targets = list(df['type']) # División en conjunto de entrenamiento y test x_train, y_train, x_test, y_test = model_creation.train_test_split( texts, targets) except: print('Error al separar en entrenamiento y pruebas') raise print('Datos separados correctamente') try: print('\n--- Creación del diccionario (Vocabulario)') tokenizer = tf_tokenizer(num_words=20000, oov_token='<null_token>', lower=False, char_level=False) tokenizer.fit_on_texts(x_train) except: print('Error al crear el vocabulario') raise print('Vocabulario creado correctamente') try: print('\n--- Transformar los textos en secuencias y padding') x_train_sequence = tokenizer.texts_to_sequences(x_train) x_test_sequence = tokenizer.texts_to_sequences(x_test) train_padded = pad_sequences(x_train_sequence, maxlen=600, dtype='int32', truncating='post', padding='post') test_padded = pad_sequences(x_test_sequence, maxlen=600, dtype='int32', truncating='post', padding='post') except: print('Error al convertir los textos en secuencias') raise print('Secuenciación y padding realizados con éxito') try: print('\n--- Definición de los hiperparámetros a optimizar') HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.2, 0.4)) METRIC_F1 = 'f1-score' with tf.summary.create_file_writer( 'logs/hparam_tuning(test11)').as_default(): hp.hparams_config( hparams=[HP_DROPOUT], metrics=[hp.Metric(METRIC_F1, display_name='F1-Score')]) except: print('Error al definir los hiperparámetros') raise print('Hiperparámetros definidos correctamente') try: print('\n--- Ejecución Grid Search') session_num = 0 for dropout_rate in (HP_DROPOUT.domain.min_value, HP_DROPOUT.domain.max_value): hparams = {HP_DROPOUT: dropout_rate} run_name = 'run-%d' % session_num print('--Iniciando ejecución : %s' % run_name) print({h: hparams[h] for h in hparams}) hparams_optimization.run('logs/hparam_tuning(test11)/' + run_name, hparams, train_padded, test_padded, y_train, y_test) session_num += 1 except: print('Error al realizar Grid Search') raise print( '\n-------------------------------------------------------------------' ) print('Grid Search realizado con éxito')