def write_parameters_table_to_tensorboard(self): """ Write the summaries, static and dynamic hyperparameters to the table in tensorboard's hparams section. This method is called once for creating the hparams table. """ # Check if needed to track hyperparameters: if (len(self._static_hyperparameters) == 0 and len(self._dynamic_hyperparameters) == 0): return # Prepare the static hyperparameters values: non_graph_parameters = {"Date": str(datetime.now()).split(".")[0]} hp_param_list = [hp_api.HParam("Date")] for parameter, value in self._static_hyperparameters.items(): non_graph_parameters[parameter] = value hp_param_list.append(hp_api.HParam(parameter)) # Prepare the summaries values and the dynamic hyperparameters values (both registered as metrics): graph_parameters = {} hp_metric_list = [] for metric in self._training_results: for prefix in ["training", "validation"]: metric_name = f"{self._Sections.SUMMARY}/{prefix}_{metric}" graph_parameters[metric_name] = 0.0 hp_metric_list.append(hp_api.Metric(metric_name)) for parameter, epochs in self._dynamic_hyperparameters.items(): parameter_name = f"{self._Sections.HYPERPARAMETERS}/{parameter}" graph_parameters[parameter_name] = epochs[-1] hp_metric_list.append(hp_api.Metric(parameter_name)) # Write the hyperparameters and summaries to the table: with self._file_writer.as_default(): hp_api.hparams_config(hparams=hp_param_list, metrics=hp_metric_list) hp_api.hparams(non_graph_parameters, trial_id=self._run_name)
def _write_hparams_config(log_dir, searchspace): HPARAMS = _create_hparams_config(searchspace) METRICS = [ hp.Metric( "epoch_accuracy", group="validation", display_name="accuracy (val.)", ), hp.Metric( "epoch_loss", group="validation", display_name="loss (val.)", ), hp.Metric( "epoch_accuracy", group="train", display_name="accuracy (train)", ), hp.Metric( "epoch_loss", group="train", display_name="loss (train)", ), ] with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config(hparams=HPARAMS, metrics=METRICS)
def log_hyperparameters(): """ Blueprint for hyperparameter and metric logging in tensorboard during hyperparameter tuning Returns: logparams (list): List containing the hyperparameters to log in tensorboard. metrics (list): List containing the metrics to log in tensorboard. """ logparams = [ hp.HParam( "latent_dim", hp.Discrete([2, 4, 6, 8, 12, 16]), display_name="latent_dim", description="encoding size dimensionality", ), hp.HParam( "n_components", hp.IntInterval(min_value=1, max_value=25), display_name="n_components", description="latent component number", ), hp.HParam( "gram_weight", hp.RealInterval(min_value=0.0, max_value=1.0), display_name="gram_weight", description="weight of the gram loss", ), ] metrics = [ hp.Metric( "val_number_of_populated_clusters", display_name="number of populated clusters", ), hp.Metric( "val_reconstruction_loss", display_name="reconstruction loss", ), hp.Metric( "val_gram_loss", display_name="gram loss", ), hp.Metric( "val_vq_loss", display_name="vq loss", ), hp.Metric( "val_total_loss", display_name="total loss", ), ] return logparams, metrics
def on_test_begin(self, logs=None): eval_writer = self._get_writer(self._validation_run_name) with eval_writer.as_default(): hp.hparams_config( hparams=[hp.HParam(k) for k, v in self.model.hparams.items()], metrics=[hp.Metric("accuracy"), hp.Metric("f1_score")], ) return super(TensorBoard, self).on_train_begin(logs=logs)
def run(hparams, logdir): with tf.summary.create_file_writer(logdir).as_default(): hp.hparams_config( hparams=[HP_HIDDEN, HP_EPOCHS, HP_LEARNING_RATE], metrics=[ hp.Metric('mean_squared_error', display_name='mse'), hp.Metric('r2', display_name='r2') ], ) mse, r2 = train_test_model(hparams, logdir) tf.summary.scalar('mean_squared_error', mse, step=1) tf.summary.scalar('r2', r2, step=1)
def main(args): del args tv, vv = FLAGS.tv, FLAGS.vv data_version = f'tv{tv}-vv{vv}' log_dir = 'tensorboard/hparam' with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config( hparams=[ HP_BATCH_SIZE, HP_OPTIMIZER, HP_LEARNING_RATE, HP_NUM_LAYERS ], metrics=[ hp.Metric(METRIC_LOSS, display_name='Entropy Loss'), hp.Metric(METRIC_MAE, display_name='MAE') ], ) for num_layers in HP_NUM_LAYERS.domain.values: for batch_size in HP_BATCH_SIZE.domain.values: for optimizer in HP_OPTIMIZER.domain.values: for learning_rate in HP_LEARNING_RATE.domain.values: trial_version = f'bs{batch_size}-{optimizer}-lr{learning_rate}-nl{num_layers}' hparam_flags = [ '--num_layers', str(num_layers), '--batch_size', str(batch_size), '--optimizer', str(optimizer), '--learning_rate', str(learning_rate), '--tv', str(tv), '--vv', str(vv), ] trial_dir = f'{log_dir}/{data_version}/{trial_version}' trial_id = f'{data_version}-{trial_version}' output = run(hparam_flags) val_loss = output['val_loss'] val_mae = output['val_mae'] hparams = { 'num_layers': num_layers, 'batch_size': batch_size, 'optimizer': optimizer, 'learning_rate': learning_rate, } with tf.summary.create_file_writer(trial_dir).as_default(): hp.hparams(hparams, trial_id) tf.summary.scalar(METRIC_LOSS, val_loss, step=1) tf.summary.scalar(METRIC_MAE, val_mae, step=1)
def hparams_combinations(hparams): hp.hparams_config(hparams=list(hparams.values()), metrics=[ hp.Metric('accuracy', display_name='Accuracy'), hp.Metric('recall', display_name='Recall'), hp.Metric('precision', display_name='Precision'), ]) hparams_keys = list(hparams.keys()) hparams_values = list(product(*[h.domain.values for h in hparams.values()])) hparams = [dict(zip(hparams_keys, values)) for values in hparams_values] shuffle(hparams) return hparams
def hparam_tuning(configs): """used to tune hyperparameters with tensorboard's hparam module currently uses the following: LATENT_DIMS = hp.HParam('latent_dim', hp.Discrete([2, 3])) RECON_WEIGHTS = hp.HParam('reconstruction_weight', hp.RealInterval(0.9, 0.99)) INTERMEDIATE_DIM = hp.HParam('intermediate_dim', hp.Discrete([12, 24, 48])) metrics displayed are: - average reconstruction loss over sample - average accuracy of latent space clustering with svm Args: configs (dict): training configs. Returns: None. """ # hyperparameters to tune logdir = "tensorboard_logs/aae/" + datetime.now().strftime("%Y%m%d-%H%M%S") with tf.summary.create_file_writer(logdir + "/hparam_tuning").as_default(): hp.hparams_config( hparams=[LATENT_DIMS, RECON_WEIGHTS, INTERMEDIATE_DIM], metrics=[ hp.Metric("acc", display_name='accuracy score'), hp.Metric("mse", display_name='reconstruction error') ], ) session_num = 0 for intermediate_dim in INTERMEDIATE_DIM.domain.values: for recon_weights in np.arange(RECON_WEIGHTS.domain.min_value, RECON_WEIGHTS.domain.max_value, 0.05): for latent_dims in LATENT_DIMS.domain.values: hparams = { INTERMEDIATE_DIM: intermediate_dim, RECON_WEIGHTS: recon_weights, LATENT_DIMS: latent_dims, } run_name = "run-%d" % session_num print('--- Starting trial: %s' % run_name) print({h.name: hparams[h] for h in hparams}) run(logdir + '/hparam_tuning/' + run_name, configs, hparams) session_num += 1
def hp_tuning(estimator_input, input_width, model_path): HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([128, 256, 512, 1024])) HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.0, 0.05, 0.1, 0.5])) HP_LEARNING_RATE = [0.00001, 0.0005, 0.001] HP_ACTIVATION = hp.HParam('activation', hp.Discrete(['relu', 'tanh'])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer(model_path + '/logs/').as_default(): hp.hparams_config( hparams=[HP_NUM_UNITS, HP_ACTIVATION], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')], ) session_num = 0 for num_units in HP_NUM_UNITS.domain.values: for dropout_rate in HP_DROPOUT.domain.values: for activation in HP_ACTIVATION.domain.values: for learning_rate in HP_LEARNING_RATE: hparams = { 'num_units': num_units, 'dropout': dropout_rate, 'activation': activation, 'learning_rate': learning_rate } run_name = id_from_hp(hparams) print('--- Starting trial: %s' % run_name) print({k: v for k, v in hparams.items()}) run(model_path + '/logs/' + run_name, hparams, estimator_input, input_width) session_num += 1
def tuning(self): self.HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([3, 6])) self.HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.1, 0.2)) self.HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd'])) self.METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[ self.HP_NUM_UNITS, self.HP_DROPOUT, self.HP_OPTIMIZER ], metrics=[ hp.Metric(self.METRIC_ACCURACY, display_name='Accuracy') ], ) session_num = 0 for num_units in self.HP_NUM_UNITS.domain.values: for dropout_rate in (self.HP_DROPOUT.domain.min_value, self.HP_DROPOUT.domain.max_value): for optimizer in self.HP_OPTIMIZER.domain.values: hparams = { self.HP_NUM_UNITS: num_units, self.HP_DROPOUT: dropout_rate, self.HP_OPTIMIZER: optimizer, } run_name = "run-%d" % session_num print('--- Starting trial: %s' % run_name) print({h.name: hparams[h] for h in hparams}) self.run('logs/hparam_tuning/' + run_name, hparams) session_num += 1
def hyperparameter_setup(self): tf.summary.trace_on(graph=True, profiler=True) with tf.summary.create_file_writer(self.log_dir).as_default(): hp.hparams_config( hparams=[HP_NH2, HP_LAMBDA], metrics=[hp.Metric(METRIC_MSE, display_name='mse')], )
def __init__(self, logdir: str, hparams: Dict[str, Union[Tuple[float, float], List]], metrics: Dict[str, str]): self._hparams = [] for name, param in hparams.items(): if isinstance(param, Tuple): min, max = param if isinstance(min, float): self._hparams.append( hp.HParam( name, hp.RealInterval(min_value=min, max_value=max))) elif isinstance(min, int): self._hparams.append( hp.HParam(name, hp.IntInterval(min_value=min, max_value=max))) elif isinstance(param, List): self._hparams.append(hp.HParam(name, hp.Discrete(param))) self._metrics = metrics self._writer = tf.summary.create_file_writer(logdir=logdir) with self._writer.as_default(): hp.hparams_config( hparams=self._hparams, metrics=[ hp.Metric(name, display_name=display) for name, display in metrics.items() ], )
def configure_hparams(self, hp_val, metrics): log.info(f"Configure hyper-params on session: {self.sess_num}") log.info(hp_val) with self.trace_writer.as_default(): hp.hparams_config( hparams=[hparam.tf_hparam for hparam in self.hp_params.values()], metrics=[ hp.Metric( f"{prefix}{eval_func}", display_name=f"{prefix}{eval_func}" ) for prefix, eval_func in list( itertools.product( *list([["TRAIN_", "VALID_", "TEST_"], metrics]) ) ) ], ) hp.hparams( dict( ( self.hp_params[param_name].tf_hparam, param_val if not self.hp_params[param_name].advanced_dtype else str(param_val), ) for param_name, param_val in hp_val.items() ) )
def hypertune_model(): """ This function uses teh previous functions to iterate through a set of parameters and run all possible combinations. ---------- Parameters: NONE ---------- """ with tf.summary.create_file_writer('logs/hparam_tuning').as_default(): hp.hparams_config( hparams=[ HP_NUM_CONV_UNITS, HP_NUM_LSTM_UNITS, HP_NUM_DENSE_UNITS, HP_DROPOUT ], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')]) session_num = 86 for num_conv_units in HP_NUM_CONV_UNITS.domain.values: for num_dense_units in HP_NUM_DENSE_UNITS.domain.values: for num_lstm_units in HP_NUM_LSTM_UNITS.domain.values: for dropout_rate in (HP_DROPOUT.domain.min_value, HP_DROPOUT.domain.max_value): hparams = { HP_NUM_CONV_UNITS: num_conv_units, HP_NUM_DENSE_UNITS: num_dense_units, HP_NUM_LSTM_UNITS: num_lstm_units, HP_DROPOUT: dropout_rate, } run_name = "run-%d" % session_num print('--- Starting trial: %s' % run_name) print({h.name: hparams[h] for h in hparams}) run('logs/hparam_tuning/' + run_name, hparams) session_num += 1
def run_all(train_dataset, val_dataset, logdir, verbose=False): """Perform random search over the hyperparameter space. Arguments: logdir: The top-level directory into which to write data. This directory should be empty or nonexistent. verbose: If true, print out each run's name as it begins. """ rng = random.Random(0) with tf.summary.create_file_writer(logdir).as_default(): hp.hparams_config( HPARAMS, metrics=[ hp.Metric( "batch_loss", group="train", display_name="loss (train)", ), hp.Metric(METRIC_LOSS, group='validation', display_name='Val loss') ], ) sessions_per_group = 1 num_sessions = 30 * sessions_per_group session_index = 0 # across all session groups for group_index in xrange(30): hparams = {h: h.domain.sample_uniform(rng) for h in HPARAMS} hparams_string = str(hparams) for repeat_index in xrange(sessions_per_group): session_id = str(session_index) session_index += 1 if verbose: print("--- Running training session %d/%d" % (session_index, num_sessions)) print(hparams_string) print("--- repeat #: %d" % (repeat_index + 1)) run( train_dataset, val_dataset, base_logdir=logdir, session_id=session_id, hparams=hparams, )
def __init__(self, num_session_groups=10, HP_CONV_LAYERS=hp.HParam("conv_layers", hp.IntInterval(1, 3)), HP_CONV_KERNEL_SIZE=hp.HParam("conv_kernel_size", hp.Discrete([3, 5])), HP_DENSE_LAYERS=hp.HParam("dense_layers", hp.IntInterval(1, 3)), HP_DROPOUT=hp.HParam("dropout", hp.RealInterval(0.1, 0.4)), HP_OPTIMIZER=hp.HParam("optimizer", hp.Discrete(["adam", "adagrad"]))): self.HP_CONV_LAYERS = HP_CONV_LAYERS self.HP_CONV_KERNEL_SIZE = HP_CONV_KERNEL_SIZE self.HP_DENSE_LAYERS = HP_DENSE_LAYERS self.HP_DROPOUT = HP_DROPOUT self.HP_OPTIMIZER = HP_OPTIMIZER self.num_session_groups = num_session_groups self.HPARAMS = [ HP_CONV_LAYERS, HP_CONV_KERNEL_SIZE, HP_DENSE_LAYERS, HP_DROPOUT, HP_OPTIMIZER, ] self.METRICS = [ hp.Metric( "epoch_accuracy", group="train", display_name="accuracy (train)", ), hp.Metric( "epoch_loss", group="train", display_name="loss (train)", ), hp.Metric( "epoch_accuracy", group="validation", display_name="accuracy (val.)", ), hp.Metric( "epoch_loss", group="validation", display_name="loss (val.)", ) ]
def evaluate(self, avpool=True): predictions = self._models["full"].predict(self._val_ds) num_categories = len(self.categories) for i in range(num_categories): category = self.categories[i] preds = predictions[:, i] y_true = self._val_labels[:, i] acc = np.mean(y_true == (preds >= 0.5).astype(int)) auc = roc_auc_score(y_true, preds) self._record_scalars(**{ f"val_accuracy_{category}": acc, f"val_auc_{category}": auc }, metric=True) # choose the hyperparameters to record if not hasattr(self, "_hparams_config"): from tensorboard.plugins.hparams import api as hp metrics = [] for c in self.categories: metrics.append(f"val_accuracy_{c}") metrics.append(f"val_auc_{c}") hparams = { hp.HParam("lam", hp.RealInterval(0., 10000.)): self.config["lam"], hp.HParam("tau", hp.RealInterval(0., 10000.)): self.config["tau"], hp.HParam("mu", hp.RealInterval(0., 10000.)): self.config["mu"], hp.HParam("batch_size", hp.RealInterval(0., 10000.)): self.input_config["batch_size"], hp.HParam("lr", hp.RealInterval(0., 10000.)): self.config["lr"], hp.HParam("lr_decay", hp.RealInterval(0., 10000.)): self.config["lr_decay"], hp.HParam("decay_type", hp.Discrete(["cosine", "exponential", "staircase"])): self.config["decay_type"], hp.HParam("opt_type", hp.Discrete(["sgd", "adam", "momentum"])): self.config["opt_type"], hp.HParam("weight_decay", hp.RealInterval(0., 10000.)): self.config["weight_decay"] } self._hparams_config = hp.hparams_config( hparams=list(hparams.keys()), metrics=[hp.Metric(m) for m in metrics]) # record hyperparameters base_dir, run_name = os.path.split(self.logdir) if len(run_name) == 0: base_dir, run_name = os.path.split(base_dir)
def _init_dir(self): try: os.mkdir(self.save_dir) except FileNotFoundError: print( 'Unable to create save_dir directory. Ensure that all but the last directory in the path already exist.' ) raise except FileExistsError: print('Directory already exists.') if self.overwrite: print('Overwriting directory.') shutil.rmtree(self.save_dir[:-1]) os.mkdir(self.save_dir) else: print('Generating new directory.') v = 2 new_path = self.save_dir while os.path.exists(new_path): new_path = self.save_dir[:-1] + '_v' + str(v) + '/' v += 1 self.save_dir = new_path os.mkdir(self.save_dir) print('New directory:\n{}'.format(self.save_dir)) assert self.save_dir[-1] == '/', \ 'Last character of self.save_dir must be \'/\', but is \'{}\''.format(self.save_dir[-1]) with tf.summary.create_file_writer(self.save_dir).as_default(): hp.hparams_config( hparams=self.hparams, metrics=[ hp.Metric(self.METRIC_TEST_ACCURACY, display_name='Test Accuracy'), hp.Metric(self.METRIC_TEST_ACCURACY_UNCERTAINTY, display_name='Test Accuracy Uncertainty'), hp.Metric(self.METRIC_TRAINING_LOSS, display_name='Training Loss'), hp.Metric(self.METRIC_VALIDATION_ACCURACY, display_name='Validation Accuracy') ])
def definitions_tensorflow(): metrics = [hp.Metric('tp'), hp.Metric('fp'), hp.Metric('tn'), hp.Metric('fn'), hp.Metric('precision'), hp.Metric('recall'), hp.Metric('auc')] return metrics
def hparams_combinations(hparams, metrics): hp.hparams_config(hparams=list(hparams.values()), metrics=[ hp.Metric(metric, display_name=metric.capitalize()) for metric in metrics ]) hparams_keys = list(hparams.keys()) hparams_values = list(product(*[h.domain.values for h in hparams.values()])) hparams = [dict(zip(hparams_keys, values)) for values in hparams_values] shuffle(hparams) return hparams
def train(): run_experiment_fn = build_experiment_fn() log_hparams_path = os.path.join(FLAGS.tb_log_dir, 'hparam_tuning') with tf.summary.create_file_writer(log_hparams_path).as_default(): hp.hparams_config(hparams=[ HP_TOKEN_TYPE, HP_VOCAB_SIZE, HP_EMBED_SIZE, HP_MEMORY_SIZE, HP_MEMORY_HOPS, HP_BATCH_SIZE, HP_EPOCHS, HP_LEARNING_RATE ], metrics=[ hp.Metric(METRIC_ACCURACY, display_name='Accuracy'), hp.Metric(METRIC_LOSS, display_name='Loss') ]) session_num = 0 for token_type in HP_TOKEN_TYPE.domain.values: for vocab_size in HP_VOCAB_SIZE.domain.values: for embedding_size in HP_EMBED_SIZE.domain.values: for memory_size in HP_MEMORY_SIZE.domain.values: for memory_hops in HP_MEMORY_HOPS.domain.values: for batch_size in HP_BATCH_SIZE.domain.values: for epochs in HP_EPOCHS.domain.values: for learning_rate in HP_LEARNING_RATE.domain.values: run_experiment_fn( session_num, { HP_TOKEN_TYPE: token_type, HP_MEMORY_SIZE: memory_size, HP_VOCAB_SIZE: vocab_size, HP_BATCH_SIZE: batch_size, HP_EMBED_SIZE: embedding_size, HP_MEMORY_HOPS: memory_hops, HP_LEARNING_RATE: learning_rate, HP_EPOCHS: epochs }) session_num += 1
def init(hp_structure, hp_dropout, hp_optimizer, hp_learningrate, hp_l2, hp_activation, BATCH_SIZES, EPOCH): HP_STRUCTURE = hp_structure HP_DROPOUT = hp_dropout HP_OPTIMIZER = hp_optimizer HP_LEARNINGRATE = hp_learningrate HP_L2 = hp_l2 HP_ACTIVATION = hp_activation batch_sizes = BATCH_SIZES epoch = EPOCH with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config( hparams=[ hp_structure, hp_dropout, hp_optimizer, HP_MOMENTUM, hp_learningrate, hp_l2, hp_activation, HP_AUGMENTATION ], metrics=[ hp.Metric(METRIC_ACCURACY, display_name='Accuracy_test'), hp.Metric(METRIC_LOSS, display_name='loss_test') ], )
def define_hparams(self, config): self.hparams_def = {} hparams = [ self.hparam_from_config_space_item(c) for c in config['space'] ] metrics = [hp.Metric('loss', display_name='loss')] with tf.summary.create_file_writer(self.log_dir + '/hparam_tuning').as_default(): hp.hparams_config( hparams=hparams, metrics=metrics, ) return self.hparams_def
def tune_hparams(epochs, train_dataset, val_dataset, HPARAMS, logdir): with tf.summary.create_file_writer(str(logdir)).as_default(): hp.hparams_config( hparams=HPARAMS, metrics=[hp.Metric('train_loss', display_name='TrainMSELoss'), hp.Metric('val_loss', display_name='ValMSELoss')] ) with open(logdir / 'config.txt', 'w') as f: content = "\n".join([param.name + ":" + str(param.domain.values) for param in HPARAMS]) f.write(content) session_num = 0 import itertools domains = [param.domain.values for param in HPARAMS] for assignment in itertools.product(*domains): hparams = {param: val for param, val in zip(HPARAMS, assignment)} print(f'--- Starting trial: run-{session_num}') print({h.name: hparams[h] for h in hparams}) model, outdir = train_once(epochs=epochs, train_dataset=train_dataset, val_dataset=val_dataset, hparams=hparams, base_log_dir=logdir) evaluate.test_on_track(model, outdir) session_num += 1
def init_hparams(hparams, FLAGS): hparam_dir = os.path.join(FLAGS.artifacts_dir, 'tblogs') os.makedirs(hparam_dir, exist_ok=True) metric = hp.Metric('sparse_categorical_accuracy', display_name='acc') with tf.summary.create_file_writer(hparam_dir).as_default(): hp.hparams_config( hparams=hparams, metrics=[metric], ) return hparam_dir
def _initialization(self, study: optuna.Study) -> None: completed_trials = [ trial for trial in study.get_trials(deepcopy=False) if trial.state == optuna.trial.TrialState.COMPLETE ] for trial in completed_trials: self._add_distributions(trial.distributions) with self._writer.as_default(): hp.hparams_config(hparams=list(self._hp_params.values()), metrics=[ hp.Metric(self._metric_name, display_name=self._metric_name) ])
def hparam_tuning(train_dataset: tf.data.Dataset, val_dataset: tf.data.Dataset, epochs: int, log_dir: str): """Performs hyperparameter tuning. Args: train_dataset: A `tf.data` dataset. Should return a tuple of `(inputs, labels)`. val_dataset: A `tf.data` dataset on which to evaluate the loss and any metrics at the end of each epoch. Should return a tuple of `(inputs, labels)`. epochs: Number of epochs to train the model. log_dir: The directory where logs will be written. """ HP_DROPOUT = hp.HParam('dropout', hp.Discrete([0.1, 0.2, 0.5])) HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([1e-3, 1e-4, 1e-5])) HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['sgd', 'rmsprop', 'adam'])) METRIC_ACCURACY = 'accuracy' with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config( hparams=[HP_DROPOUT, HP_LEARNING_RATE, HP_OPTIMIZER], metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')]) trial_step = 0 for dropout in HP_DROPOUT.domain.values: for learning_rate in HP_LEARNING_RATE.domain.values: for optimizer in HP_OPTIMIZER.domain.values: hparams = { HP_DROPOUT: dropout, HP_LEARNING_RATE: learning_rate, HP_OPTIMIZER: optimizer, } trial_id = f'run-{trial_step}' trial_dir = os.path.join(log_dir, trial_id) logging.info(f'--- Starting trial: {trial_id}') logging.info({h.name: hparams[h] for h in hparams}) accuracy = train_keras(train_dataset, val_dataset, dropout, learning_rate, optimizer, epochs, trial_dir, hparams) write_trial_log(METRIC_ACCURACY, accuracy, hparams, trial_dir) trial_step += 1
def __init__(self, log_dir, ckpt_dir, initial_model_path, custom_model_fn, train_ds, val_ds, metric): self.log_dir = log_dir self.ckpt_dir = ckpt_dir make_directory(log_dir) make_directory(ckpt_dir) self.metric = metric self.initial_model_path = initial_model_path self.train_ds = train_ds self.val_ds = val_ds self.custom_model_fn = custom_model_fn with tf.summary.create_file_writer(log_dir).as_default(): hp.hparams_config( hparams=make_hparam_list(), metrics=[hp.Metric(metric, display_name=metric)], )
def create_hparams_callback(log_dir, opt_metric, hparams, args): """ Set up Hprams plugin config and callback for Tensorboard """ hparams_dir = os.path.join(log_dir, 'validation') opt_metric = 'epoch_' + opt_metric # Hparams callback to log the hyperparameter values with tf.summary.create_file_writer(hparams_dir).as_default(): hp.hparams_config(hparams=[hp.HParam(hparam) for hparam in hparams], metrics=[hp.Metric(opt_metric)]) hparams_cb = hp.KerasCallback( writer=hparams_dir, hparams={hparam: args[hparam] for hparam in hparams}) return hparams_cb
def search(self, data_zip, groups, folds=5, positive_weight=1.0, epochs=40): hparam_tuning_fn = f"{self.run_dir_prefix}/hparam_tuning" with tf.summary.create_file_writer(hparam_tuning_fn).as_default(): hp.hparams_config( hparams=list(self.hparams_domains.values()), metrics=[hp.Metric("auc", display_name='Average AUC')] ) num = 0 best_auc = 0 best_params = None for run_num, hparam_run in enumerate(self.get_hparams_combinations()): auc = self.run_log_cv(hparam_run, run_num, data_zip, groups, folds, positive_weight, epochs) if auc > best_auc: best_params = hparam_run best_auc = auc logging.info("[Hyperparameter Detective] Best AVG AUC is %s with hparams %s", best_auc, best_params)