def test_sqa_storage(self):
init_test_engine_and_session_factory(force_init=True)
config = SQAConfig()
encoder = Encoder(config=config)
decoder = Decoder(config=config)
db_settings = DBSettings(encoder=encoder, decoder=decoder)
ax_client = AxClient(db_settings=db_settings)
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
for _ in range(5):
parameters, trial_index = ax_client.get_next_trial()
ax_client.complete_trial(trial_index=trial_index,
raw_data=branin(*parameters.values()))
gs = ax_client.generation_strategy
ax_client = AxClient(db_settings=db_settings)
ax_client.load_experiment_from_database("test_experiment")
# Trial #4 was completed after the last time the generation strategy
# generated candidates, so pre-save generation strategy was not
# "aware" of completion of trial #4. Post-restoration generation
# strategy is aware of it, however, since it gets restored with most
# up-to-date experiment data. Do adding trial #4 to the seen completed
# trials of pre-storage GS to check their equality otherwise.
gs._seen_trial_indices_by_status[TrialStatus.COMPLETED].add(4)
self.assertEqual(gs, ax_client.generation_strategy)
with self.assertRaises(ValueError):
# Overwriting existing experiment.
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
with self.assertRaises(ValueError):
# Overwriting existing experiment with overwrite flag with present
# DB settings. This should fail as we no longer allow overwriting
# experiments stored in the DB.
ax_client.create_experiment(
name="test_experiment",
parameters=[{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
}],
overwrite_existing_experiment=True,
)
# Original experiment should still be in DB and not have been overwritten.
self.assertEqual(len(ax_client.experiment.trials), 5)
def test_overwrite(self):
init_test_engine_and_session_factory(force_init=True)
ax_client = AxClient()
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
# Log a trial
parameters, trial_index = ax_client.get_next_trial()
ax_client.complete_trial(trial_index=trial_index,
raw_data=branin(*parameters.values()))
with self.assertRaises(ValueError):
# Overwriting existing experiment.
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
# Overwriting existing experiment with overwrite flag.
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 15.0]
},
],
overwrite_existing_experiment=True,
)
# There should be no trials, as we just put in a fresh experiment.
self.assertEqual(len(ax_client.experiment.trials), 0)
# Log a trial
parameters, trial_index = ax_client.get_next_trial()
self.assertIn("x1", parameters.keys())
self.assertIn("x2", parameters.keys())
ax_client.complete_trial(trial_index=trial_index,
raw_data=branin(*parameters.values()))
def finetune(optimize_consistency, evaluate_on, original_dev_dataset,
runs_per_trial, hyperparam_opt_runs, out_file, mute,
baseline_gold_file, hyperparams, keep_predictions,
original_ans_length, **kwargs):
gold_files = get_baseline_intervention_control_from_baseline(
baseline_gold_file)
golds = tuple(load_json(g) for g in gold_files)
# load eval gold for evaluation
aligneds = align(*golds, assert_same=True)
hyper_params = [{
'name': hp['name'],
'type': hp.get("type", 'range'),
'bounds': hp['bounds'],
'value_type': hp.get('value_type', 'float'),
'log_scale': hp.get('log_scale', True)
} for hp in json.loads(hyperparams)]
logger.info(hyper_params)
args = Args(**kwargs)
args.debug_features = not mute
tokenizer = get_tokenizer(args.model_path, args.do_lower_case)
features = []
for f in gold_files:
args.eval_file = f
features.append(load_or_convert(args, tokenizer, evaluate=True))
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
kwargs['n_gpu'] = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
kwargs['n_gpu'] = 1
kwargs['device'] = device
args.n_gpu = kwargs['n_gpu']
args.device = kwargs['device']
if args.seed:
set_seed(args)
logger.debug(args)
if args.fp16:
try:
import apex
apex.amp.register_half_function(torch, "einsum")
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
# load train dataset
train_dataset, train_examples, train_features = load_or_convert(
args, tokenizer)
if not mute:
debug_features_examples_dataset(train_dataset, train_examples,
train_features, tokenizer)
if original_dev_dataset:
args.eval_file = original_dev_dataset
original_dev_dataset = load_or_convert(args, tokenizer, evaluate=True)
ax_client = AxClient()
ax_client.create_experiment(
name=f'{args.model_path}@{args.train_file}',
parameters=hyper_params,
objective_name=evaluate_on,
minimize=False,
)
result = {
"trials": [],
"tried_params":
defaultdict(list),
"best_params":
...,
'pre_eval':
train_and_eval_single_step(args,
train_dataset,
*aligneds,
*features,
original_dev_dataset,
*gold_files,
run_nr='eval',
train=False,
evaluate_on=evaluate_on,
original_ans_length=original_ans_length)
}
# first, eval and save what is the performance before training
click.echo(f"Results: {json.dumps(result['pre_eval'], indent=4)}")
# run hyperparam optimisation
predictions_folder = keep_predictions
for i in trange(hyperparam_opt_runs):
parameters, trial_index = ax_client.get_next_trial()
logger.info(f"Trying parameters: {parameters}")
single_step_args = deepcopy(kwargs)
single_step_args.update(parameters)
args = Args(**single_step_args)
args.predictions_folder = str(predictions_folder)
trial_result = train_and_eval_single_step(
args,
train_dataset,
*aligneds,
*features,
original_dev_dataset,
*gold_files,
run_nr=i,
num_runs=runs_per_trial,
evaluate_on=evaluate_on,
original_ans_length=original_ans_length)
#
if optimize_consistency:
assert evaluate_on == 'eoi'
mean = trial_result['consistency']
else:
mean = trial_result['overall' if evaluate_on ==
'eoi' else 'EMRelaxed']
if runs_per_trial > 1:
mean, var, ci = mean
if original_dev_dataset:
logger.info(f"Mean: ({mean} * 100 + {trial_result['original']})/2")
mean = (mean * 100 + trial_result['original']) / 2
trial_result["mean"] = mean
logger.info(f"Result: {mean}")
logger.info(f"Results: {json.dumps(trial_result, indent=4)}")
result["trials"].append(trial_result)
result['tried_params'][i].append(parameters)
ax_client.complete_trial(trial_index=trial_index, raw_data=mean)
best_params, metrics = ax_client.get_best_parameters()
result['best_params'] = best_params
result['best_metrics'] = metrics
click.echo(f"What is metrics? {metrics}")
click.echo(json.dumps(result, indent=4))
write_json(result, out_file)
class AxSearchJob(AutoSearchJob):
"""Job for hyperparameter search using [ax](https://ax.dev/)."""
def __init__(self, config: Config, dataset, parent_job=None):
super().__init__(config, dataset, parent_job)
self.num_trials = self.config.get("ax_search.num_trials")
self.num_sobol_trials = self.config.get("ax_search.num_sobol_trials")
self.ax_client: AxClient = None
if self.__class__ == AxSearchJob:
for f in Job.job_created_hooks:
f(self)
# Overridden such that instances of search job can be pickled to workers
def __getstate__(self):
state = super(AxSearchJob, self).__getstate__()
del state["ax_client"]
return state
def init_search(self):
if self.num_sobol_trials > 0:
# BEGIN: from /ax/service/utils/dispatch.py
generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL,
num_arms=self.num_sobol_trials,
min_arms_observed=ceil(
self.num_sobol_trials / 2),
enforce_num_arms=True,
model_kwargs={'seed': 0}),
GenerationStep(
model=Models.GPEI,
num_arms=-1,
recommended_max_parallelism=3,
model_gen_kwargs={
"fixed_features":
ObservationFeatures(
parameters={
kv["name"]: kv["value"]
for kv in self.config.get(
"ax_search.fixed_parameters")
})
},
),
],
)
# END: from /ax/service/utils/dispatch.py
self.ax_client = AxClient(generation_strategy=generation_strategy)
else:
self.ax_client = AxClient()
self.ax_client.create_experiment(
name=self.job_id,
parameters=self.config.get("ax_search.parameters"),
objective_name="metric_value",
minimize=False,
parameter_constraints=self.config.get(
"ax_search.parameter_constraints"),
choose_generation_strategy_kwargs={'random_seed': 0},
)
self.config.log("ax search initialized with {}".format(
self.ax_client.generation_strategy))
# Make sure sobol models are resumed correctly
if self.ax_client.generation_strategy._curr.model == Models.SOBOL:
self.ax_client.generation_strategy._set_current_model(
experiment=self.ax_client.experiment, data=None)
# Regenerate and drop SOBOL arms already generated. Since we fixed the seed,
# we will skip exactly the arms already generated in the job being resumed.
num_generated = len(self.parameters)
if num_generated > 0:
num_sobol_generated = min(
self.ax_client.generation_strategy._curr.num_arms,
num_generated)
for i in range(num_sobol_generated):
generator_run = self.ax_client.generation_strategy.gen(
experiment=self.ax_client.experiment)
# self.config.log("Skipped parameters: {}".format(generator_run.arms))
self.config.log(
"Skipped {} of {} Sobol trials due to prior data.".format(
num_sobol_generated,
self.ax_client.generation_strategy._curr.num_arms,
))
def register_trial(self, parameters=None):
trial_id = None
try:
if parameters is None:
parameters, trial_id = self.ax_client.get_next_trial()
else:
_, trial_id = self.ax_client.attach_trial(parameters)
except Exception as e:
self.config.log(
"Cannot generate trial parameters. Will try again after a " +
"running trial has completed. message was: {}".format(e))
return parameters, trial_id
def register_trial_result(self, trial_id, parameters, trace_entry):
if trace_entry is None:
self.ax_client.log_trial_failure(trial_index=trial_id)
else:
self.ax_client.complete_trial(trial_index=trial_id,
raw_data=trace_entry["metric_value"])
def get_best_parameters(self):
best_parameters, values = self.ax_client.get_best_parameters()
return best_parameters, float(values[0]["metric_value"])
def run_ax_search(
fixed_params: Dict,
ax_params: List[Dict[str, Any]],
eval_fn: Callable,
obj_name: str,
minimize: bool,
id_: str,
parse_params_fn: Optional[Callable] = None,
ax_param_constraints: Optional[List[str]] = None,
num_ax_steps: int = 50,
num_concur_samples: int = 2,
num_seeds: int = 10,
num_proc: int = 20,
folder_name: Optional[str] = None,
verbose: bool = False,
) -> Tuple[Dict[str, Any], AxClient]:
"""
Run a search for best hyperparameter values using Ax.
Note that this requires the Ax package (https://ax.dev/) to be installed.
Args:
fixed_params: Fixed values of hyperparameters.
ax_params: Ax configuration for hyperparameters that are searched over. See docs for ax_client.create_experiment()
eval_fn: Evaluation function that returns a dictionary of metric values.
obj_name: Objective name (key of the dict reterned by eval_fn)
minimize: If True, objective is minimized, if False it's maximized.
id_: An arbitrary string identifier of the search (used as part of filename where results are saved)
parse_params_fn: A function applied to the parameter dictionary to parse it. Can be used
if the best represenation for Ax doesn't match the format accepted by the eval_fn.
ax_param_constraints: Constraints for the parameters that are searched over.
num_ax_steps: The number of ax steps to take.
num_concur_samples: Number of configurations to sample per ax step (in parallel)
num_seeds: Number of seeds to average over
num_proc: Number of processes to run in parallel.
folder_name: Folder where to save best found parameters
verbose: If True, some details are printed out
Returns:
A dict of best hyperparameters found by Ax
"""
for p in ax_params:
assert (
p["name"] not in fixed_params
), f'Parameter {p["name"]} appers in both fixed and search parameters'
if ax_param_constraints is None:
ax_param_constraints = []
ax_client = AxClient()
ax_client.create_experiment(
name=f"hparams_search_{id_}",
parameters=ax_params,
objective_name=obj_name,
minimize=minimize,
parameter_constraints=ax_param_constraints,
choose_generation_strategy_kwargs={
"max_parallelism_override":
num_concur_samples,
"num_initialization_trials":
max(num_concur_samples, 5, len(ax_params)),
},
)
best_params = None
try:
for i in range(1, num_ax_steps + 1):
if verbose:
print(f"ax step {i}/{num_ax_steps}")
params_list = []
trial_indices_list = []
for _ in range(num_concur_samples):
# sample several values (to be evaluated in parallel)
parameters, trial_index = ax_client.get_next_trial()
params_list.append(parameters)
trial_indices_list.append(trial_index)
res = ax_evaluate_params(
params_list,
fixed_params=fixed_params,
eval_fn=eval_fn,
parse_params_fn=parse_params_fn,
num_seeds=num_seeds,
num_proc=num_proc,
)
for t_i, v in zip(trial_indices_list, res):
ax_client.complete_trial(trial_index=t_i, raw_data=v)
best_params, predicted_metrics = ax_client.get_best_parameters()
predicted_metrics = predicted_metrics[
0] # choose expected metric values
if verbose:
print(best_params, predicted_metrics)
# save at every iteration in case search is interrupted
if folder_name is not None:
with open(
os.path.join(
folder_name,
f"ax_results_{id_}.json",
),
"w",
) as f:
json.dump(
{
"fixed_params": fixed_params,
"best_params": best_params,
"predicted_metrics": predicted_metrics,
},
f,
indent=4,
)
except KeyboardInterrupt:
# handle keyboard interruption to enable returning intermediate results if interrupted
pass
return best_params, ax_client
def _benchmark_replication_Service_API(
problem: SimpleBenchmarkProblem,
method: GenerationStrategy,
num_trials: int,
experiment_name: str,
batch_size: int = 1,
raise_all_exceptions: bool = False,
benchmark_trial: FunctionType = benchmark_trial,
verbose_logging: bool = True,
# Number of trials that need to fail for a replication to be considered failed.
failed_trials_tolerated: int = 5,
async_benchmark_options: Optional[AsyncBenchmarkOptions] = None,
) -> Tuple[Experiment, List[Exception]]:
"""Run a benchmark replication via the Service API because the problem was
set up in a simplified way, without the use of Ax classes like `OptimizationConfig`
or `SearchSpace`.
"""
if async_benchmark_options is not None:
raise NonRetryableBenchmarkingError(
"`async_benchmark_options` not supported when using the Service API."
)
exceptions = []
if batch_size == 1:
ax_client = AxClient(generation_strategy=method,
verbose_logging=verbose_logging)
else: # pragma: no cover, TODO[T53975770]
assert batch_size > 1, "Batch size of 1 or greater is expected."
raise NotImplementedError(
"Batched benchmarking on `SimpleBenchmarkProblem`-s not yet implemented."
)
ax_client.create_experiment(
name=experiment_name,
parameters=problem.domain_as_ax_client_parameters(),
minimize=problem.minimize,
objective_name=problem.name,
)
parameter_names = list(ax_client.experiment.search_space.parameters.keys())
assert num_trials > 0
for _ in range(num_trials):
parameterization, idx = ax_client.get_next_trial()
param_values = np.array(
[parameterization.get(x) for x in parameter_names])
try:
mean, sem = benchmark_trial(parameterization=param_values,
evaluation_function=problem.f)
# If problem indicates a noise level and is using a synthetic callable,
# add normal noise to the measurement of the mean.
if problem.uses_synthetic_function and problem.noise_sd != 0.0:
noise = np.random.randn() * problem.noise_sd
sem = (sem or 0.0) + problem.noise_sd
logger.info(
f"Adding noise of {noise} to the measurement mean ({mean})."
f"Problem noise SD setting: {problem.noise_sd}.")
mean = mean + noise
ax_client.complete_trial(trial_index=idx, raw_data=(mean, sem))
except Exception as err: # TODO[T53975770]: test
if raise_all_exceptions:
raise
exceptions.append(err)
if len(exceptions) > failed_trials_tolerated:
raise RuntimeError( # TODO[T53975770]: test
f"More than {failed_trials_tolerated} failed for {experiment_name}."
)
return ax_client.experiment, exceptions
def main(data_path, experiment_path, load_model, ratio_known_normal,
ratio_known_outlier, seed, optimizer_name, validation, lr, n_epochs,
lr_milestone, batch_size, weight_decay, pretrain, ae_optimizer_name,
ae_lr, ae_n_epochs, ae_lr_milestone, ae_batch_size, ae_weight_decay,
num_threads, n_jobs_dataloader, normal_class, known_outlier_class,
n_known_outlier_classes):
ray.init(address='auto')
data_path = os.path.abspath(data_path)
n_splits = 4
period = np.array([
'2019-11-08', '2019-11-09', '2019-11-11', '2019-11-12', '2019-11-13',
'2019-11-14', '2019-11-15'
])
dates = period[:2]
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
exp_config = {**locals().copy(), 'objective': 'soft-boundary'}
device = torch.device('cpu')
exp_config = {
**locals().copy(),
'net_name': 'cicflow_mlp',
}
if exp_config['seed'] != -1:
random.seed(exp_config['seed'])
np.random.seed(exp_config['seed'])
torch.manual_seed(exp_config['seed'])
torch.cuda.manual_seed(exp_config['seed'])
torch.backends.cudnn.deterministic = True
ax = AxClient(enforce_sequential_optimization=False)
ax.create_experiment(
name="IsoForestCICFlowExp",
parameters=[
{
"name": "n_estimators",
"type": "range",
"bounds": [100, 1000],
},
{
"name": "max_samples",
"type": "range",
"bounds": [1e-6, 1.0],
"log_scale": True
},
{
"name": "contamination",
"type": "range",
"bounds": [0.0, 0.15]
},
],
objective_name="val_auc_pr",
)
search_alg = AxSearch(ax)
re_search_alg = Repeater(search_alg, repeat=n_splits)
sched = ASHAScheduler(time_attr='training_iteration',
grace_period=10,
metric="val_auc_pr")
analysis = tune.run(IsoForestCICFlowExp,
name="OneDayIsoForestCICFlowExp",
checkpoint_at_end=True,
checkpoint_freq=5,
stop={
"training_iteration": 1,
},
resources_per_trial={"cpu": 4},
num_samples=20,
local_dir=experiment_path,
search_alg=re_search_alg,
scheduler=sched,
config=exp_config)
print("Best config is:", analysis.get_best_config(metric="val_auc_pr"))
def test_create_experiment(self) -> None:
"""Test basic experiment creation."""
ax = AxClient(
GenerationStrategy(steps=[GenerationStep(model=Models.SOBOL, num_arms=30)])
)
with self.assertRaisesRegex(ValueError, "Experiment not set on Ax client"):
ax.experiment
ax.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [0.001, 0.1],
"value_type": "float",
"log_scale": True,
},
{
"name": "x2",
"type": "choice",
"values": [1, 2, 3],
"value_type": "int",
"is_ordered": True,
},
{"name": "x3", "type": "fixed", "value": 2, "value_type": "int"},
{
"name": "x4",
"type": "range",
"bounds": [1.0, 3.0],
"value_type": "int",
},
{
"name": "x5",
"type": "choice",
"values": ["one", "two", "three"],
"value_type": "str",
},
{
"name": "x6",
"type": "range",
"bounds": [1.0, 3.0],
"value_type": "int",
},
],
objective_name="test_objective",
minimize=True,
outcome_constraints=["some_metric >= 3", "some_metric <= 4.0"],
parameter_constraints=["x4 <= x6"],
)
assert ax._experiment is not None
self.assertEqual(ax._experiment, ax.experiment)
self.assertEqual(
ax._experiment.search_space.parameters["x1"],
RangeParameter(
name="x1",
parameter_type=ParameterType.FLOAT,
lower=0.001,
upper=0.1,
log_scale=True,
),
)
self.assertEqual(
ax._experiment.search_space.parameters["x2"],
ChoiceParameter(
name="x2",
parameter_type=ParameterType.INT,
values=[1, 2, 3],
is_ordered=True,
),
)
self.assertEqual(
ax._experiment.search_space.parameters["x3"],
FixedParameter(name="x3", parameter_type=ParameterType.INT, value=2),
)
self.assertEqual(
ax._experiment.search_space.parameters["x4"],
RangeParameter(
name="x4", parameter_type=ParameterType.INT, lower=1.0, upper=3.0
),
)
self.assertEqual(
ax._experiment.search_space.parameters["x5"],
ChoiceParameter(
name="x5",
parameter_type=ParameterType.STRING,
values=["one", "two", "three"],
),
)
self.assertEqual(
ax._experiment.optimization_config.outcome_constraints[0],
OutcomeConstraint(
metric=Metric(name="some_metric"),
op=ComparisonOp.GEQ,
bound=3.0,
relative=False,
),
)
self.assertEqual(
ax._experiment.optimization_config.outcome_constraints[1],
OutcomeConstraint(
metric=Metric(name="some_metric"),
op=ComparisonOp.LEQ,
bound=4.0,
relative=False,
),
)
self.assertTrue(ax._experiment.optimization_config.objective.minimize)
def automatic_hyperparameter_search(drug_id,
num_trails=50,
num_splits=3,
num_epochs=50,
data_folder='../example_data/'):
data_file_name = f'{data_folder}/{drug_id}.pickle'
with open(data_file_name, 'rb') as f:
data = pickle.load(f)
if 'hyperparameter_search' in data.keys():
return
ax_client = AxClient()
ax_client.create_experiment(name=f"drug_id_{drug_id}",
parameters=[
{
"name": "lr",
"value_type": 'float',
"type": "range",
"bounds": [1e-5, 1e0],
"log_scale": True
},
{
"name":
"l2_regularization_coefficient",
"value_type": 'float',
"type": "range",
"bounds": [1e-5, 1e0],
"log_scale": True
},
{
"name": "num_layers",
"value_type": 'int',
"type": "range",
"bounds": [1, 5]
},
{
"name": "num_neurons",
"value_type": 'int',
"type": "range",
"bounds": [10, 100]
},
],
objective_name="score",
minimize=False)
for i in range(num_trails):
parameters, trial_index = ax_client.get_next_trial()
ax_client.complete_trial(trial_index=trial_index,
raw_data=evaluate(parameters, data,
num_splits, num_epochs))
best_parameters, values = ax_client.get_best_parameters()
trace = ax_client.get_optimization_trace()
data['hyperparameter_search'] = {}
data['hyperparameter_search']['score'] = values[0]['score']
data['hyperparameter_search']['neural_net_config'] = best_parameters
with open(data_file_name, 'wb') as f:
pickle.dump(data, f)
class AxSearch(Searcher):
"""Uses `Ax <https://ax.dev/>`_ to optimize hyperparameters.
Ax is a platform for understanding, managing, deploying, and
automating adaptive experiments. Ax provides an easy to use
interface with BoTorch, a flexible, modern library for Bayesian
optimization in PyTorch. More information can be found in https://ax.dev/.
To use this search algorithm, you must install Ax and sqlalchemy:
.. code-block:: bash
$ pip install ax-platform sqlalchemy
Parameters:
space (list[dict]): Parameters in the experiment search space.
Required elements in the dictionaries are: "name" (name of
this parameter, string), "type" (type of the parameter: "range",
"fixed", or "choice", string), "bounds" for range parameters
(list of two values, lower bound first), "values" for choice
parameters (list of values), and "value" for fixed parameters
(single value).
objective_name (str): Name of the metric used as objective in this
experiment. This metric must be present in `raw_data` argument
to `log_data`. This metric must also be present in the dict
reported/returned by the Trainable.
mode (str): One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute. Defaults to "max".
parameter_constraints (list[str]): Parameter constraints, such as
"x3 >= x4" or "x3 + x4 >= 2".
outcome_constraints (list[str]): Outcome constraints of form
"metric_name >= bound", like "m1 <= 3."
ax_client (AxClient): Optional AxClient instance. If this is set, do
not pass any values to these parameters: `space`, `objective_name`,
`parameter_constraints`, `outcome_constraints`.
use_early_stopped_trials: Deprecated.
max_concurrent (int): Deprecated.
Tune automatically converts search spaces to Ax's format:
.. code-block:: python
from ray import tune
from ray.tune.suggest.ax import AxSearch
config = {
"x1": tune.uniform(0.0, 1.0),
"x2": tune.uniform(0.0, 1.0)
}
def easy_objective(config):
for i in range(100):
intermediate_result = config["x1"] + config["x2"] * i
tune.report(score=intermediate_result)
ax_search = AxSearch(objective_name="score")
tune.run(
config=config,
easy_objective,
search_alg=ax_search)
If you would like to pass the search space manually, the code would
look like this:
.. code-block:: python
from ray import tune
from ray.tune.suggest.ax import AxSearch
parameters = [
{"name": "x1", "type": "range", "bounds": [0.0, 1.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 1.0]},
]
def easy_objective(config):
for i in range(100):
intermediate_result = config["x1"] + config["x2"] * i
tune.report(score=intermediate_result)
ax_search = AxSearch(space=parameters, objective_name="score")
tune.run(easy_objective, search_alg=ax_search)
"""
def __init__(self,
space: Optional[Union[Dict, List[Dict]]] = None,
metric: Optional[str] = None,
mode: Optional[str] = None,
parameter_constraints: Optional[List] = None,
outcome_constraints: Optional[List] = None,
ax_client: Optional[AxClient] = None,
use_early_stopped_trials: Optional[bool] = None,
max_concurrent: Optional[int] = None):
assert ax is not None, "Ax must be installed!"
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
super(AxSearch,
self).__init__(metric=metric,
mode=mode,
max_concurrent=max_concurrent,
use_early_stopped_trials=use_early_stopped_trials)
self._ax = ax_client
if isinstance(space, dict) and space:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
if domain_vars or grid_vars:
logger.warning(
UNRESOLVED_SEARCH_SPACE.format(par="space",
cls=type(self)))
space = self.convert_search_space(space)
self._space = space
self._parameter_constraints = parameter_constraints
self._outcome_constraints = outcome_constraints
self.max_concurrent = max_concurrent
self._objective_name = metric
self._parameters = []
self._live_trial_mapping = {}
if self._ax or self._space:
self.setup_experiment()
def setup_experiment(self):
if not self._ax:
self._ax = AxClient()
try:
exp = self._ax.experiment
has_experiment = True
except ValueError:
has_experiment = False
if not has_experiment:
if not self._space:
raise ValueError(
"You have to create an Ax experiment by calling "
"`AxClient.create_experiment()`, or you should pass an "
"Ax search space as the `space` parameter to `AxSearch`, "
"or pass a `config` dict to `tune.run()`.")
self._ax.create_experiment(
parameters=self._space,
objective_name=self._metric,
parameter_constraints=self._parameter_constraints,
outcome_constraints=self._outcome_constraints,
minimize=self._mode != "max")
else:
if any([
self._space, self._parameter_constraints,
self._outcome_constraints
]):
raise ValueError(
"If you create the Ax experiment yourself, do not pass "
"values for these parameters to `AxSearch`: {}.".format([
"space", "parameter_constraints", "outcome_constraints"
]))
exp = self._ax.experiment
self._objective_name = exp.optimization_config.objective.metric.name
self._parameters = list(exp.parameters)
if self._ax._enforce_sequential_optimization:
logger.warning("Detected sequential enforcement. Be sure to use "
"a ConcurrencyLimiter.")
def set_search_properties(self, metric: Optional[str], mode: Optional[str],
config: Dict):
if self._ax:
return False
space = self.convert_search_space(config)
self._space = space
if metric:
self._metric = metric
if mode:
self._mode = mode
self.setup_experiment()
return True
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._ax:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self.max_concurrent:
if len(self._live_trial_mapping) >= self.max_concurrent:
return None
parameters, trial_index = self._ax.get_next_trial()
self._live_trial_mapping[trial_id] = trial_index
return unflatten_dict(parameters)
def on_trial_complete(self, trial_id, result=None, error=False):
"""Notification for the completion of trial.
Data of form key value dictionary of metric names and values.
"""
if result:
self._process_result(trial_id, result)
self._live_trial_mapping.pop(trial_id)
def _process_result(self, trial_id, result):
ax_trial_index = self._live_trial_mapping[trial_id]
metric_dict = {
self._objective_name: (result[self._objective_name], 0.0)
}
outcome_names = [
oc.metric.name for oc in
self._ax.experiment.optimization_config.outcome_constraints
]
metric_dict.update({on: (result[on], 0.0) for on in outcome_names})
self._ax.complete_trial(trial_index=ax_trial_index,
raw_data=metric_dict)
@staticmethod
def convert_search_space(spec: Dict):
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to an Ax search space.")
def resolve_value(par, domain):
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
logger.warning("AxSearch does not support quantization. "
"Dropped quantization.")
sampler = sampler.sampler
if isinstance(domain, Float):
if isinstance(sampler, LogUniform):
return {
"name": par,
"type": "range",
"bounds": [domain.lower, domain.upper],
"value_type": "float",
"log_scale": True
}
elif isinstance(sampler, Uniform):
return {
"name": par,
"type": "range",
"bounds": [domain.lower, domain.upper],
"value_type": "float",
"log_scale": False
}
elif isinstance(domain, Integer):
if isinstance(sampler, LogUniform):
return {
"name": par,
"type": "range",
"bounds": [domain.lower, domain.upper],
"value_type": "int",
"log_scale": True
}
elif isinstance(sampler, Uniform):
return {
"name": par,
"type": "range",
"bounds": [domain.lower, domain.upper],
"value_type": "int",
"log_scale": False
}
elif isinstance(domain, Categorical):
if isinstance(sampler, Uniform):
return {
"name": par,
"type": "choice",
"values": domain.categories
}
raise ValueError("AxSearch does not support parameters of type "
"`{}` with samplers of type `{}`".format(
type(domain).__name__,
type(domain.sampler).__name__))
# Fixed vars
fixed_values = [{
"name": "/".join(path),
"type": "fixed",
"value": val
} for path, val in resolved_vars]
# Parameter name is e.g. "a/b/c" for nested dicts
resolved_values = [
resolve_value("/".join(path), domain)
for path, domain in domain_vars
]
return fixed_values + resolved_values
def test_sqa_storage(self):
init_test_engine_and_session_factory(force_init=True)
config = SQAConfig()
encoder = Encoder(config=config)
decoder = Decoder(config=config)
db_settings = DBSettings(encoder=encoder, decoder=decoder)
ax_client = AxClient(db_settings=db_settings)
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
for _ in range(5):
parameters, trial_index = ax_client.get_next_trial()
ax_client.complete_trial(trial_index=trial_index,
raw_data=branin(*parameters.values()))
gs = ax_client.generation_strategy
ax_client = AxClient(db_settings=db_settings)
ax_client.load_experiment_from_database("test_experiment")
self.assertEqual(gs, ax_client.generation_strategy)
with self.assertRaises(ValueError):
# Overwriting existing experiment.
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
with self.assertRaises(ValueError):
# Overwriting existing experiment with overwrite flag with present
# DB settings. This should fail as we no longer allow overwriting
# experiments stored in the DB.
ax_client.create_experiment(
name="test_experiment",
parameters=[{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
}],
overwrite_existing_experiment=True,
)
# Original experiment should still be in DB and not have been overwritten.
self.assertEqual(len(ax_client.experiment.trials), 5)
if (option == "tuning"):
return vallossmean, sem
else:
return res, mcp_save, model
else:
print("GPU is not available")
init_notebook_plotting()
ax_client = AxClient()
# create the experiment.
ax_client.create_experiment(name="keras_experiment",
parameters=parameters,
objective_name='keras_cv',
minimize=True)
def evaluate(parameters):
return {
"keras_cv":
training(parameters, "tuning", X_train_partial[train_index],
y_train_partial[train_index], X_train_partial[val_index],
y_train_partial[val_index])
}
for i in range(25):
skf = StratifiedKFold(n_splits=10, random_state=7, shuffle=True)
for train_index, val_index in skf.split(np.zeros(X_train_partial.shape[0]),
class IterativePrune:
def __init__(self):
self.parser_args = None
self.ax_client = None
self.base_model_path = "base_model"
self.pruning_amount = None
def run_mnist_model(self, base=False):
parser_dict = vars(self.parser_args)
if base:
mlflow.start_run(run_name="BaseModel")
mlflow.pytorch.autolog()
dm = MNISTDataModule(**parser_dict)
dm.prepare_data()
dm.setup(stage="fit")
model = LightningMNISTClassifier(**parser_dict)
trainer = pl.Trainer.from_argparse_args(self.parser_args)
trainer.fit(model, dm)
trainer.test()
if os.path.exists(self.base_model_path):
shutil.rmtree(self.base_model_path)
mlflow.pytorch.save_model(trainer.get_model(), self.base_model_path)
return trainer
def load_base_model(self):
path = Path(_download_artifact_from_uri(self.base_model_path))
model_file_path = os.path.join(path, "data/model.pth")
return torch.load(model_file_path)
def initialize_ax_client(self):
self.ax_client = AxClient()
self.ax_client.create_experiment(
parameters=[{
"name": "amount",
"type": "range",
"bounds": [0.05, 0.15],
"value_type": "float"
}],
objective_name="test_accuracy",
)
@staticmethod
def prune_and_save_model(model, amount):
for _, module in model.named_modules():
if isinstance(module, torch.nn.Conv2d) or isinstance(
module, torch.nn.Linear):
prune.l1_unstructured(module, name="weight", amount=amount)
prune.remove(module, "weight")
mlflow.pytorch.save_state_dict(model.state_dict(), ".")
model = torch.load("state_dict.pth")
os.remove("state_dict.pth")
return model
@staticmethod
def count_model_parameters(model):
table = PrettyTable(["Modules", "Parameters"])
total_params = 0
for name, parameter in model.named_parameters():
if not parameter.requires_grad:
continue
param = parameter.nonzero(as_tuple=False).size(0)
table.add_row([name, param])
total_params += param
return table, total_params
@staticmethod
def write_prune_summary(summary, params):
tempdir = tempfile.mkdtemp()
try:
summary_file = os.path.join(tempdir, "pruned_model_summary.txt")
params = "Total Trainable Parameters :" + str(params)
with open(summary_file, "w") as f:
f.write(str(summary))
f.write("\n")
f.write(str(params))
try_mlflow_log(mlflow.log_artifact, local_path=summary_file)
finally:
shutil.rmtree(tempdir)
def iterative_prune(self, model, parametrization):
if not self.pruning_amount:
self.pruning_amount = parametrization.get("amount")
else:
self.pruning_amount += 0.15
mlflow.log_metric("PRUNING PERCENTAGE", self.pruning_amount)
pruned_model = self.prune_and_save_model(model, self.pruning_amount)
model.load_state_dict(copy.deepcopy(pruned_model))
summary, params = self.count_model_parameters(model)
self.write_prune_summary(summary, params)
trainer = self.run_mnist_model()
metrics = trainer.callback_metrics
test_accuracy = metrics.get("avg_test_acc")
return test_accuracy
def initiate_pruning_process(self, model):
total_trials = int(vars(self.parser_args)["total_trials"])
trial_index = None
for i in range(total_trials):
parameters, trial_index = self.ax_client.get_next_trial()
print(
"***************************************************************************"
)
print("Running Trial {}".format(i + 1))
print(
"***************************************************************************"
)
with mlflow.start_run(nested=True, run_name="Iteration" + str(i)):
mlflow.set_tags({"AX_TRIAL": i})
# calling the model
test_accuracy = self.iterative_prune(model, parameters)
# completion of trial
self.ax_client.complete_trial(trial_index=trial_index,
raw_data=test_accuracy.item())
# Ending the Base run
mlflow.end_run()
def get_parser_args(self):
parser = argparse.ArgumentParser()
parser = pl.Trainer.add_argparse_args(parent_parser=parser)
parser = LightningMNISTClassifier.add_model_specific_args(
parent_parser=parser)
parser.add_argument(
"--total_trials",
default=3,
help=
"Number of AX trials to be run for the optimization experiment",
)
self.parser_args = parser.parse_args()
"name": "x4",
"type": "range",
"bounds": [0.0, 1.0],
},
{
"name": "x5",
"type": "range",
"bounds": [0.0, 1.0],
},
{
"name": "x6",
"type": "range",
"bounds": [0.0, 1.0],
},
]
client = AxClient(enforce_sequential_optimization=False)
client.create_experiment(
parameters=parameters,
objective_name="hartmann6",
minimize=True, # Optional, defaults to False.
parameter_constraints=["x1 + x2 <= 2.0"], # Optional.
outcome_constraints=["l2norm <= 1.25"], # Optional.
)
algo = AxSearch(client, max_concurrent=4)
scheduler = AsyncHyperBandScheduler(metric="hartmann6", mode="max")
run(easy_objective,
name="ax",
search_alg=algo,
scheduler=scheduler,
**config)
def test_default_generation_strategy(self) -> None:
"""Test that Sobol+GPEI is used if no GenerationStrategy is provided."""
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 15.0]
},
],
objective_name="branin",
minimize=True,
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL, Models.GPEI],
)
with self.assertRaisesRegex(ValueError, ".* no trials."):
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
for i in range(6):
parameterization, trial_index = ax_client.get_next_trial()
x1, x2 = parameterization.get("x1"), parameterization.get("x2")
ax_client.complete_trial(
trial_index,
raw_data={
"branin": (
checked_cast(
float,
branin(checked_cast(float, x1),
checked_cast(float, x2)),
),
0.0,
)
},
sample_size=i,
)
if i < 5:
with self.assertRaisesRegex(ValueError,
"Could not obtain contour"):
ax_client.get_contour_plot(param_x="x1", param_y="x2")
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
ax_client.get_contour_plot()
self.assertIn("x1", ax_client.get_trials_data_frame())
self.assertIn("x2", ax_client.get_trials_data_frame())
self.assertIn("branin", ax_client.get_trials_data_frame())
self.assertEqual(len(ax_client.get_trials_data_frame()), 6)
# Test that Sobol is chosen when all parameters are choice.
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{
"name": "x1",
"type": "choice",
"values": [1, 2, 3]
},
{
"name": "x2",
"type": "choice",
"values": [1, 2, 3]
},
])
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL],
)
self.assertEqual(ax_client.get_recommended_max_parallelism(),
[(-1, -1)])
self.assertTrue(ax_client.get_trials_data_frame().empty)
def testConvertAx(self):
from ray.tune.suggest.ax import AxSearch
from ax.service.ax_client import AxClient
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = AxSearch.convert_search_space(config)
ax_config = [
{
"name": "a",
"type": "choice",
"values": [2, 3, 4]
},
{
"name": "b/x",
"type": "range",
"bounds": [0, 5],
"value_type": "int"
},
{
"name": "b/y",
"type": "fixed",
"value": 4
},
{
"name": "b/z",
"type": "range",
"bounds": [1e-4, 1e-2],
"value_type": "float",
"log_scale": True
},
]
client1 = AxClient(random_seed=1234)
client1.create_experiment(parameters=converted_config)
searcher1 = AxSearch(ax_client=client1, metric="a", mode="max")
client2 = AxClient(random_seed=1234)
client2.create_experiment(parameters=ax_config)
searcher2 = AxSearch(ax_client=client2, metric="a", mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertEqual(config1["b"]["y"], 4)
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = AxSearch(metric="a", mode="max")
analysis = tune.run(
_mock_objective, config=config, search_alg=searcher, num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {"a": tune.uniform(5, 6), "b": tune.uniform(8, 9)}
searcher = AxSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def test_sqa_storage(self):
init_test_engine_and_session_factory(force_init=True)
config = SQAConfig()
encoder = Encoder(config=config)
decoder = Decoder(config=config)
db_settings = DBSettings(encoder=encoder, decoder=decoder)
ax_client = AxClient(db_settings=db_settings)
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
for _ in range(5):
parameters, trial_index = ax_client.get_next_trial()
ax_client.complete_trial(trial_index=trial_index,
raw_data=branin(*parameters.values()))
gs = ax_client.generation_strategy
ax_client = AxClient(db_settings=db_settings)
ax_client.load_experiment_from_database("test_experiment")
self.assertEqual(gs, ax_client.generation_strategy)
with self.assertRaises(ValueError):
# Overwriting existing experiment.
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 15.0]
},
],
minimize=True,
)
# Overwriting existing experiment with overwrite flag.
ax_client.create_experiment(
name="test_experiment",
parameters=[{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
}],
overwrite_existing_experiment=True,
)
# There should be no trials, as we just put in a fresh experiment.
self.assertEqual(len(ax_client.experiment.trials), 0)
class AxInterface(BaseInterface):
"""Specific override to support the Ax backend -- supports the service style API from Ax"""
def __init__(self, tuner_config: AxTunerConfig, tuner_namespace):
"""AxInterface init call that maps variables, creates a map to fnc calls, and constructs the necessary
underlying objects
Args:
tuner_config: configuration object for the ax backend
tuner_namespace: tuner namespace that has attr classes that maps to an underlying library types
"""
super(AxInterface, self).__init__(tuner_config, tuner_namespace)
self._tuner_obj = AxClient(
generation_strategy=self._tuner_config.generation_strategy,
enforce_sequential_optimization=self._tuner_config.
enforce_sequential_optimization,
random_seed=self._tuner_config.random_seed,
verbose_logging=self._tuner_config.verbose_logging,
)
# Some variables to use later
self._trial_index = None
self._sample_hash = None
# Mapping spock underlying classes to ax distributions (search space)
self._map_type = {
"RangeHyperParameter": {
"int": self._ax_range,
"float": self._ax_range,
},
"ChoiceHyperParameter": {
"int": self._ax_choice,
"float": self._ax_choice,
"str": self._ax_choice,
"bool": self._ax_choice,
},
}
# Build the correct underlying dictionary object for Ax client create experiment
self._param_obj = self._construct()
# Create the AxClient experiment
self._tuner_obj.create_experiment(
parameters=self._param_obj,
name=self._tuner_config.name,
objective_name=self._tuner_config.objective_name,
minimize=self._tuner_config.minimize,
parameter_constraints=self._tuner_config.parameter_constraints,
outcome_constraints=self._tuner_config.outcome_constraints,
overwrite_existing_experiment=self._tuner_config.
overwrite_existing_experiment,
tracking_metric_names=self._tuner_config.tracking_metric_names,
immutable_search_space_and_opt_config=self._tuner_config.
immutable_search_space_and_opt_config,
is_test=self._tuner_config.is_test,
)
@property
def tuner_status(self) -> AxTunerStatus:
return AxTunerStatus(client=self._tuner_obj,
trial_index=self._trial_index)
@property
def best(self):
best_obj = self._tuner_obj.get_best_parameters()
rollup_dict, _ = self._sample_rollup(best_obj[0])
return (
self._gen_spockspace(rollup_dict),
best_obj[1][0][self._tuner_obj.objective_name],
)
@property
def _get_sample(self):
return self._tuner_obj.get_next_trial()
def sample(self):
parameters, self._trial_index = self._get_sample
# Roll this back out into a Spockspace so it can be merged into the fixed parameter Spockspace
# Also need to un-dot the param names to rebuild the nested structure
rollup_dict, sample_hash = self._sample_rollup(parameters)
self._sample_hash = sample_hash
return self._gen_spockspace(rollup_dict)
def _construct(self):
param_list = []
# These will only be nested one level deep given the tuner syntax
for k, v in vars(self._tuner_namespace).items():
for ik, iv in vars(v).items():
param_fn = self._map_type[type(iv).__name__][iv.type]
param_list.append(param_fn(name=f"{k}.{ik}", val=iv))
return param_list
def _ax_range(self, name, val):
"""Assemble the dictionary for ax range parameters
Args:
name: parameter name
val: current attr val
Returns:
dictionary that can be added to a parameter list
"""
low, high = self._try_range_cast(val,
type_string="RangeHyperParameter")
return {
"name": name,
"type": "range",
"bounds": [low, high],
"value_type": val.type,
"log_scale": val.log_scale,
}
def _ax_choice(self, name, val):
"""Assemble the dictionary for ax choice parameters
Args:
name: parameter name
val: current attr val
Returns:
dictionary that can be added to a parameter list
"""
val = self._try_choice_cast(val, type_string="ChoiceHyperParameter")
return {
"name": name,
"type": "choice",
"values": val.choices,
"value_type": val.type,
}
for workload in workloads:
for aware in awares:
if aware:
awareness = 'aware'
else :
awareness = 'blind'
ax_client = AxClient()
runner = Runner(workload, aware)
ax_client.create_experiment(
name=f"{workload}_{awareness}_experiment",
parameters=[
{
"name": "alpha",
"type": "range",
"bounds": [0.01, 100],
"value_type": "float", # Optional, defaults to inference from type of "bounds".
"log_scale": True,
}
],
objective_name="p99",
minimize=True # Optional, defaults to False.
)
# pretrain on earlier results - These are commented out below, down to the "run the opti loop"
#results_dir = Path('.')
#exp_params = [ cdf_gen_lib_v2.getExperimentParamaters(os.path.join(results_dir, exp), quiet=True) for exp in os.listdir(results_dir)]
#exp_params = list(filter(None, exp_params))
#exp_params = list(filter(lambda exp_param: exp_param.is_dsalt and f'W{exp_param.workload}' == workload and exp_param.is_aware == aware, exp_params))
#batch_params = {}
