def test_same_set_of_points_ask(strategy, surrogate):
"""
For n_points not None, tests whether two consecutive calls to ask
return the same sets of points.
Parameters
----------
* `strategy` [string]:
Name of the strategy to use during optimization.
* `surrogate` [scikit-optimize surrogate class]:
A class of the scikit-optimize surrogate used in Optimizer.
"""
optimizer = Optimizer(
base_estimator=surrogate(),
dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)],
acq_optimizer='sampling',
random_state=2
)
for i in range(n_steps):
xa = optimizer.ask(n_points, strategy)
xb = optimizer.ask(n_points, strategy)
optimizer.tell(xa, [branin(v) for v in xa])
assert_equal(xa, xb) # check if the sets of points generated are equal
def test_constant_liar_runs(strategy, surrogate):
"""
Tests whether the optimizer runs properly during the random
initialization phase and beyond
Parameters
----------
* `strategy` [string]:
Name of the strategy to use during optimization.
* `surrogate` [scikit-optimize surrogate class]:
A class of the scikit-optimize surrogate used in Optimizer.
"""
optimizer = Optimizer(
base_estimator=surrogate(),
dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)],
acq_optimizer='sampling',
random_state=0
)
# test arguments check
assert_raises(ValueError, optimizer.ask, {"strategy": "cl_maen"})
assert_raises(ValueError, optimizer.ask, {"n_points": "0"})
assert_raises(ValueError, optimizer.ask, {"n_points": 0})
for i in range(n_steps):
x = optimizer.ask(n_points=n_points, strategy=strategy)
# check if actually n_points was generated
assert_equal(len(x), n_points)
optimizer.tell(x, [branin(v) for v in x])
def test_dict_list_space_representation():
"""
Tests whether the conversion of the dictionary and list representation
of a point from a search space works properly.
"""
chef_space = {
'Cooking time': (0, 1200), # in minutes
'Main ingredient': [
'cheese', 'cherimoya', 'chicken', 'chard', 'chocolate', 'chicory'
],
'Secondary ingredient': [
'love', 'passion', 'dedication'
],
'Cooking temperature': (-273.16, 10000.0) # in Celsius
}
opt = Optimizer(dimensions=dimensions_aslist(chef_space))
point = opt.ask()
# check if the back transformed point and original one are equivalent
assert_equal(
point,
point_aslist(chef_space, point_asdict(chef_space, point))
)
def test_all_points_different(strategy, surrogate):
"""
Tests whether the parallel optimizer always generates
different points to evaluate.
Parameters
----------
* `strategy` [string]:
Name of the strategy to use during optimization.
* `surrogate` [scikit-optimize surrogate class]:
A class of the scikit-optimize surrogate used in Optimizer.
"""
optimizer = Optimizer(
base_estimator=surrogate(),
dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)],
acq_optimizer='sampling',
random_state=1
)
tolerance = 1e-3 # distance above which points are assumed same
for i in range(n_steps):
x = optimizer.ask(n_points, strategy)
optimizer.tell(x, [branin(v) for v in x])
distances = pdist(x)
assert all(distances > tolerance)
def test_dump_and_load_optimizer():
base_estimator = ExtraTreesRegressor(random_state=2)
opt = Optimizer([(-2.0, 2.0)], base_estimator, n_random_starts=1,
acq_optimizer="sampling")
opt.run(bench1, n_iter=3)
with tempfile.TemporaryFile() as f:
dump(opt, f)
f.seek(0)
load(f)
def test_reproducible_runs(strategy, surrogate):
# two runs of the optimizer should yield exactly the same results
optimizer = Optimizer(
base_estimator=surrogate(random_state=1),
dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)],
acq_optimizer='sampling',
random_state=1
)
points = []
for i in range(n_steps):
x = optimizer.ask(n_points, strategy)
points.append(x)
optimizer.tell(x, [branin(v) for v in x])
# the x's should be exaclty as they are in `points`
optimizer = Optimizer(
base_estimator=surrogate(random_state=1),
dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)],
acq_optimizer='sampling',
random_state=1
)
for i in range(n_steps):
x = optimizer.ask(n_points, strategy)
assert points[i] == x
optimizer.tell(x, [branin(v) for v in x])
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'):
self.__all_experiments = pd.DataFrame()
self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments)
self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN')))
self.__all_experiments['client'] = [""] * len(self.__all_experiments)
self.__lease_duration = pd.to_timedelta(lease_timout)
self.__leased_experiments = []
dims = self.__load_dimensions(dimensions_file)
self.__dimension_names = list(dims.keys())
self.__dimensions = list(dims.values())
self.__min_num_results_to_fit = min_num_results_to_fit
# Initialize
dim_types = [check_dimension(d) for d in self.__dimensions]
is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types])
if is_cat:
transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions]
else:
transformed_dims = []
for dim_type, dim in zip(dim_types, self.__dimensions):
if isinstance(dim_type, Categorical):
transformed_dims.append(check_dimension(dim, transform="onehot"))
# To make sure that GP operates in the [0, 1] space
else:
transformed_dims.append(check_dimension(dim, transform="normalize"))
space = Space(transformed_dims)
# Default GP
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims))
acq_optimizer = "lbfgs"
else:
other_kernel = Matern(
length_scale=np.ones(space.transformed_n_dims),
length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, random_state=None, alpha=0.0, noise='gaussian',
n_restarts_optimizer=2)
self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs",
n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
def get_optimizer(config, features):
return Optimizer(
[
Categorical(categories=[0, 1], name=f, prior=[0.75, 0.25])
for f in features
],
#[Integer(low=0, high=1, name=f) for f in features],
base_estimator=config.get('base_estimator', 'ET'),
acq_optimizer=config.get('acq_optimizer', 'sampling'),
n_random_starts=config.get('n_random_starts', None),
n_initial_points=config.get('n_initial_points', 10),
acq_func=config.get('acq_func', 'gp_hedge'),
random_state=config.get('random_state', None),
acq_func_kwargs=config.get('acq_func_kwargs', None),
acq_optimizer_kwargs=config.get('acq_optimizer_kwargs', None))
def optimizer(self):
"""Return skopt's optimizer."""
if self._optimizer is None:
self._optimizer = Optimizer(
base_estimator=GaussianProcessRegressor(
alpha=self.alpha,
n_restarts_optimizer=self.n_restarts_optimizer,
noise=self.noise,
normalize_y=self.normalize_y),
dimensions=orion_space_to_skopt_space(self.space),
n_initial_points=self.n_initial_points,
acq_func=self.acq_func)
self.seed_rng(self.seed)
return self._optimizer
def setup_optz():
logger.debug("setup() START")
from skopt import Optimizer
# Set up
problem = Problem()
starting_point = problem.starting_point
seed = 42
# Start the optimizer
parDict = { 'kappa' : 1.96 }
space = [problem.space[key] for key in problem.params]
optimizer = Optimizer(space, base_estimator='RF', acq_optimizer='sampling',
acq_func='LCB', acq_func_kwargs={}, random_state=seed)
logger.debug("setup() STOP")
return (problem, optimizer)
def set_optimizer(self, n_iter, opt_seed, acq_func, gp_seed, **kwargs):
self.logger.info('Retrieving model stored at: {}'.format(self.optimizer_path))
try:
optimizer = load(self.optimizer_path)
self.logger.info('Loading model stored at: {}'.format(self.optimizer_path))
finished_iter = np.array(optimizer.yi).shape[0]
if finished_iter == 0:
optimizer = None
self.logger.info('Optimizer did not finish any iterations so setting optimizer to null')
except KeyError:
self.logger.error('Cannot open the file {}'.format(self.optimizer_path))
optimizer = None
except ValueError:
self.logger.error('Cannot open the file {}'.format(self.optimizer_path))
optimizer = None
except FileNotFoundError:
self.logger.error('No such file or directory: {}'.format(self.optimizer_path))
optimizer = None
if optimizer is not None:
n_iter = n_iter - finished_iter
if n_iter < 0:
n_iter = 0
self.logger.info('Iterations already done: {} and running iterations {}'.format(finished_iter, n_iter))
self.opt = optimizer
self.logger.debug('Setting the provided optimizer')
self.log_best_params()
else:
transformed = []
for param in self.parameter_ranges:
transformed.append(check_dimension(param))
self.logger.info("Parameter Space: {}".format(transformed))
norm_space = normalize_dimensions(transformed)
self.logger.info("Parameter Space after transformation: {}".format(norm_space))
categorical_space = np.array([isinstance(s, Categorical) for s in norm_space])
self.logger.info("categorical_space: {}".format(categorical_space))
if np.all(categorical_space):
base_estimator = cook_estimator("RF", space=norm_space, random_state=gp_seed)
else:
base_estimator = cook_estimator("GP", space=norm_space, random_state=gp_seed, noise="gaussian")
self.opt = Optimizer(dimensions=self.parameter_ranges, random_state=opt_seed, base_estimator=base_estimator,
acq_func=acq_func, **kwargs)
return n_iter
def create_opt(lines, ranker_name):
gp_seed, opt_seed = get_seed(lines)
_ranker_class = object_rankers[ranker_name]
_ranker_class._use_early_stopping = True
param_ranges = _ranker_class.set_tunable_parameter_ranges({})
transformed = []
for param in param_ranges:
transformed.append(check_dimension(param))
space = normalize_dimensions(transformed)
base_estimator = cook_estimator("GP",
space=space,
random_state=gp_seed,
noise="gaussian")
optimizer = Optimizer(dimensions=param_ranges,
random_state=opt_seed,
base_estimator=base_estimator)
return optimizer
def run_batch(s):
f = open(
"/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_args",
"rb")
args = pickle.load(f)
f = open(
"/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_spaces",
"rb")
hyperspaces = pickle.load(f)
for i in s:
current_space = hyperspaces[i]
optimizer = Optimizer(current_space)
if args.model == "segmentation_cityscapes" or args.model == "segmentation_gis":
search_algo = SkOptSearch(
optimizer,
['learning_rate', 'epochs', 'batch_size', 'adam_epsilon'],
metric='average_res',
mode='max')
else:
search_algo = SkOptSearch(optimizer, [
'learning_rate', 'dropout', 'epochs', 'batch_size',
'adam_epsilon'
],
metric='average_res',
mode='max')
analysis = tune.run(
multi_train,
search_alg=search_algo,
num_samples=int(args.trials),
resources_per_trial={
'cpu': 25,
'gpu': 1
},
local_dir="/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/ray_results"
)
df = analysis.results_df
df_name = "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out + "/"
df_name += "space_"
df_name += str(i)
df_name += ".csv"
df.to_csv(df_name)
print("Finished space " + args.space)
print(
"Finished all spaces. Files writtten to /lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/"
+ args.out)
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(n_jobs)
# Initialize Hyperaparameter space
self.hp_space = []
# add the 'learning_rate' space to the HPO search space
self.hp_space.append(self.problem.space["hyperparameters"]["learning_rate"])
# add the 'batch_size' space to the HPO search space
self.hp_space.append(self.problem.space["hyperparameters"]["batch_size"])
# Initialize opitmizer of hyperparameter space
acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration
# self.free_workers = 128 #! TODO: test
self.n_initial_points = self.free_workers
self.hp_opt = SkOptimizer(
dimensions=self.hp_space,
base_estimator=RandomForestRegressor(n_jobs=32),
# base_estimator=RandomForestRegressor(n_jobs=4),
acq_func="LCB",
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
# model_queue_size=100,
)
def __init__(self,
problem,
num_workers,
surrogate_model='RF',
acq_func='gp_hedge',
acq_kappa=1.96,
liar_strategy='cl_max',
n_jobs=1,
**kwargs):
assert surrogate_model in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.space = problem.space
# queue of remaining starting points
self.starting_points = problem.starting_point
n_init = inf if surrogate_model == 'DUMMY' else max(
num_workers, len(self.starting_points))
self._optimizer = SkOptimizer(self.space.values(),
base_estimator=base_estimator,
acq_optimizer='sampling',
acq_func=acq_func,
acq_func_kwargs={'kappa': acq_kappa},
random_state=self.SEED,
n_initial_points=n_init)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info(
f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__(self,
api_config,
base_estimator="GP",
acq_func="gp_hedge",
n_initial_points=5,
**kwargs):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
base_estimator : {'GP', 'RF', 'ET', 'GBRT'}
How to estimate the objective function.
acq_func : {'LCB', 'EI', 'PI', 'gp_hedge', 'EIps', 'PIps'}
Acquisition objective to decide next suggestion.
n_initial_points : int
Number of points to sample randomly before actual Bayes opt.
"""
AbstractOptimizer.__init__(self, api_config)
dimensions, self.round_to_values = ScikitOptimizer.get_sk_dimensions(
api_config)
# Older versions of skopt don't copy over the dimensions names during
# normalization and hence the names are missing in
# self.skopt.space.dimensions. Therefore, we save our own copy of
# dimensions list to be safe. If we can commit to using the newer
# versions of skopt we can delete self.dimensions.
self.dimensions_list = tuple(dd.name for dd in dimensions)
# Undecided where we want to pass the kwargs, so for now just make sure
# they are blank
assert len(kwargs) == 0
self.skopt = SkOpt(
dimensions,
n_initial_points=n_initial_points,
base_estimator=base_estimator,
acq_func=acq_func,
acq_optimizer="auto",
acq_func_kwargs={},
acq_optimizer_kwargs={},
)
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
kappa=0.001,
xi=0.000001,
acq_func="LCB",
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
# Initialize Hyperaparameter space
self.hp_space = self.problem._hp_space
# Initialize opitmizer of hyperparameter space
acq_func_kwargs = {
"xi": float(xi),
"kappa": float(kappa)
} # tiny exploration
self.n_initial_points = self.free_workers
self.hp_opt = SkOptimizer(
dimensions=self.hp_space._space,
base_estimator=RandomForestRegressor(n_jobs=self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
)
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
acq_optimizer = "sampling"
if isinstance(estimator, str):
if estimator not in ("GP", "RF", "ET", "GBRT"):
raise OperationalException(f"Estimator {estimator} not supported.")
else:
acq_optimizer = "auto"
logger.info(f"Using estimator {estimator}.")
return Optimizer(
dimensions,
base_estimator=estimator,
acq_optimizer=acq_optimizer,
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.random_state,
model_queue_size=SKOPT_MODEL_QUEUE_SIZE,
)
def __init__(self, problem, num_workers, args):
assert args.learner in ["RF", "ET", "GBRT", "GP", "DUMMY"], f"Unknown scikit-optimize base_estimator: {args.learner}"
self.space = problem.space
n_init = inf if args.learner=='DUMMY' else num_workers
self._optimizer = SkOptimizer(
self.space.values(),
base_estimator=args.learner,
acq_optimizer='sampling',
acq_func=args.acq_func,
acq_func_kwargs={'kappa':self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init
)
assert args.liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = args.liar_strategy
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" % args.learner)
def search(self,
data: Dataset,
metrics: List[str],
cv: Any,
n_jobs: int,
verbose: int = 0) -> ResultGroup:
"""
Perform a bayesian search over the specified hyperparameters
Parameters
----------
data: Dataset
Instance of data to train on
metrics: List of str
List of metrics to calculate results for
cv: Any
Either a CV object from sklearn or an int to specify number of folds
n_jobs: int
Number of jobs to calculate in parallel
verbose: int
Verbosity level of the method
Returns
-------
ResultGroup
"""
optimizer = Optimizer(dimensions_aslist(self.param_grid))
logger.info("Starting Bayesian search...")
results = [
self._step(optimizer, data, metrics, cv, n_jobs, verbose)
for _ in range(self.n_iter)
]
logger.info("Finished Bayesian search...")
return ResultGroup(results).sort()
def _make_optimizer(self, params_space):
"""Instantiate skopt Optimizer class.
Parameters
----------
params_space : dict
Represents parameter search space. The keys are parameter
names (strings) and values are skopt.space.Dimension instances,
one of Real, Integer or Categorical.
Returns
-------
optimizer: Instance of the `Optimizer` class used for for search
in some parameter space.
"""
kwargs = self.optimizer_kwargs_.copy()
kwargs['dimensions'] = dimensions_aslist(params_space)
optimizer = Optimizer(**kwargs)
return optimizer
def submit(n, optimizer: Optimizer, opt_param_names, current_configs,
param_space: ParamSpace, queue: Queue):
""" Generate and submit n new configurations to a queue.
Asks the optimizer for n new values to explore, creates configurations for those points and puts them
in the given queue.
Args:
n: the number of configurations to be generated
optimizer: the optimiser object from skopt with the model used for the suggested points to explore
opt_param_names: the names for the parameters using the same order of the dimensions in the optimizer
current_configs: current list of configurations (updated with the newly generated ones)
param_space: parameter space which we can use to convert optimizer points to fully specified configurations
queue: que multiprocessing queue in which we put the new configurations
"""
dims = opt_param_names
xs = optimizer.ask(n_points=n)
cfgs = [values_to_params(dict(zip(dims, x)), param_space) for x in xs]
for i, c in enumerate(cfgs):
c["id"] = i + len(current_configs)
queue.put(c)
current_configs += cfgs
def create_model(self, bounds, modeltag=''):
"""Updates the model by pickling and uploading (overwriting) the opt model to a special async bucket generated by the stack
:param bounds: UUID of the model
:type bounds: list of tuples, like [(-1.0,1.0),(-5,5),(-10.5,10.5)...]
:param modeltag: Friendly name of the optimizer
:type modeltag: string, optional
"""
self.modelname = str(uuid4())
if all(isinstance(item, tuple) for item in bounds):
logging.info('Creating model with bounds: ' + str(bounds))
opt = Optimizer(bounds,
"GP",
acq_func="EI",
acq_optimizer="sampling",
initial_point_generator="lhs")
self.update_model(self.modelname, opt)
logging.info('created model with name ' + self.modelname)
if modeltag != '':
s3_client = boto3.client('s3')
response = s3_client.put_object_tagging(
Bucket=self.asyncbucket,
Key=self.modelname + '/model.pkl',
Tagging={'TagSet': [
{
'Key': 'tag',
'Value': modeltag
},
]})
return self.modelname
else:
logging.error(
'Input bounds as a list of tuples, like [(-2.0, 2.0), ...]')
class Optimizer:
SEED = 12345
KAPPA = 1.96
def __init__(self, num_workers: int, space, learner, acq_func,
liar_strategy, **kwargs):
assert learner in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {learner}"
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.space = space
self.learner = learner
self.acq_func = acq_func
self.liar_strategy = liar_strategy
n_init = inf if learner == 'DUMMY' else num_workers
self._optimizer = SkOptimizer(dimensions=self.space.dimensions,
base_estimator=self.learner,
acq_optimizer='sampling',
acq_func=self.acq_func,
acq_func_kwargs={'kappa': self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init)
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" %
self.learner)
def _get_lie(self):
if self.liar_strategy == "cl_min":
return min(self._optimizer.yi) if self._optimizer.yi else 0.0
elif self.liar_strategy == "cl_mean":
return np.mean(self._optimizer.yi) if self._optimizer.yi else 0.0
else:
return max(self._optimizer.yi) if self._optimizer.yi else 0.0
def _xy_from_dict(self):
XX = list(self.evals.keys())
YY = [self.evals[x] for x in XX]
return XX, YY
def to_dict(self, x: list) -> dict:
return self.space.to_dict(x)
def _ask(self):
x = self._optimizer.ask()
y = self._get_lie()
key = tuple(x)
if key not in self.evals:
self.counter += 1
self._optimizer.tell(x, y)
self.evals[key] = y
logger.debug(f'_ask: {x} lie: {y}')
else:
logger.debug(f'Duplicate _ask: {x} lie: {y}')
return self.to_dict(x)
def ask(self, n_points=None, batch_size=20):
if n_points is None:
return self._ask()
else:
batch = []
for _ in range(n_points):
batch.append(self._ask())
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
def ask_initial(self, n_points):
XX = self._optimizer.ask(n_points=n_points)
for x in XX:
y = self._get_lie()
key = tuple(x)
if key not in self.evals:
self.counter += 1
self._optimizer.tell(x, y)
self.evals[key] = y
return [self.to_dict(x) for x in XX]
def tell(self, xy_data):
assert isinstance(xy_data,
list), f"where type(xy_data)=={type(xy_data)}"
maxval = max(self._optimizer.yi) if self._optimizer.yi else 0.0
for x, y in xy_data:
key = tuple(x.values()) # * tuple(x[k] for k in self.space)
assert key in self.evals, f"where key=={key} and self.evals=={self.evals}"
logger.debug(f'tell: {x} --> {key}: evaluated objective: {y}')
self.evals[key] = (y if y < float_info.max else maxval)
self._optimizer.Xi = []
self._optimizer.yi = []
XX, YY = self._xy_from_dict()
assert len(XX) == len(YY) == self.counter, (
f"where len(XX)=={len(XX)},"
f"len(YY)=={len(YY)}, self.counter=={self.counter}")
self._optimizer.tell(XX, YY)
assert len(self._optimizer.Xi) == len(
self._optimizer.yi) == self.counter, (
f"where len(self._optimizer.Xi)=={len(self._optimizer.Xi)}, "
f"len(self._optimizer.yi)=={len(self._optimizer.yi)},"
f"self.counter=={self.counter}")
class BayesianOptimizedExperimentQueue(ExperimentQueue):
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'):
self.__all_experiments = pd.DataFrame()
self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments)
self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN')))
self.__all_experiments['client'] = [""] * len(self.__all_experiments)
self.__lease_duration = pd.to_timedelta(lease_timout)
self.__leased_experiments = []
dims = self.__load_dimensions(dimensions_file)
self.__dimension_names = list(dims.keys())
self.__dimensions = list(dims.values())
self.__min_num_results_to_fit = min_num_results_to_fit
# Initialize
dim_types = [check_dimension(d) for d in self.__dimensions]
is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types])
if is_cat:
transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions]
else:
transformed_dims = []
for dim_type, dim in zip(dim_types, self.__dimensions):
if isinstance(dim_type, Categorical):
transformed_dims.append(check_dimension(dim, transform="onehot"))
# To make sure that GP operates in the [0, 1] space
else:
transformed_dims.append(check_dimension(dim, transform="normalize"))
space = Space(transformed_dims)
# Default GP
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims))
acq_optimizer = "lbfgs"
else:
other_kernel = Matern(
length_scale=np.ones(space.transformed_n_dims),
length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, random_state=None, alpha=0.0, noise='gaussian',
n_restarts_optimizer=2)
self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs",
n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
@property
def all_experiments(self) -> pd.DataFrame:
"""
:return: The PandasFrame containing the details for all the experiments in the queue.
"""
return self.__all_experiments
@property
def completed_percent(self) -> float:
return 0.
@property
def leased_percent(self) -> float:
return 0
@property
def experiment_parameters(self) -> List:
return self.__dimension_names
def lease_new(self, client_name: str) -> Tuple[int, Dict]:
"""
Lease a new experiment lock. Select first any waiting experiments and then re-lease expired ones
:param client_name: The name of the leasing client
:return: a tuple (id, parameters) or None if nothing is available
"""
experiment_params = self.__opt.ask()
if experiment_params in self.__leased_experiments:
experiment_params = self.__compute_alternative_params()
self.__leased_experiments.append(experiment_params)
# TODO: Add to all experiments, use Ids
def parse_dim_val(value, dim_type):
if type(dim_type) is Real:
return float(value)
elif type(dim_type) is Integer:
return int(value)
return value
return {name: parse_dim_val(value, dim_type) for name, dim_type, value in zip(self.__dimension_names, self.__dimensions, experiment_params)}, -1
def __compute_alternative_params(self):
# Copied directly from skopt
transformed_bounds = np.array(self.__opt.space.transformed_bounds)
est = clone(self.__opt.base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.__opt.space.transform(self.__opt.Xi), self.__opt.yi)
X = self.__opt.space.transform(self.__opt.space.rvs(
n_samples=self.__opt.n_points, random_state=self.__opt.rng))
values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi),
acq_func='EI',
acq_func_kwargs=dict(n_points=10000))
print('original point ei: %s' % np.min(values))
discount_width = .5
values = self.__discount_leased_params(X, values, discount_width)
while np.min(values) > -1e-5 and discount_width > 1e-2:
discount_width *= .9
values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi),
acq_func='EI',
acq_func_kwargs=dict(n_points=10000))
values = self.__discount_leased_params(X, values, discount_width)
next_x = X[np.argmin(values)]
print('new point ei: %s' % np.min(values))
if not self.__opt.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0], transformed_bounds[:, 1])
return self.__opt.space.inverse_transform(next_x.reshape((1, -1)))[0]
@staticmethod
def leased_discount(center, width, x_values):
"""Triangular (cone) discount"""
distance_from_center = np.linalg.norm(x_values - center, 2, axis=1)
discount = -distance_from_center / width + 1
discount[discount < 0] = 0
return discount
def __discount_leased_params(self, X, values, discount_width_size):
transformed_leased_params = self.__opt.space.transform(np.array(self.__leased_experiments))
discount_factor = reduce(lambda x, y: x * y,
(self.leased_discount(p, discount_width_size, X) for p in self.__leased_experiments),
np.ones(values.shape[0]))
out_vals = values * (1. - discount_factor)
return out_vals
def complete(self, experiment_id: int, parameters: Dict, client: str, result: float = 0) -> None:
"""
Declare an experiment to be completed.
:param experiment_id: the id of the experiment or -1 if unknown
:param client: the client id
:param result: the output results of the experiment. This may be used in optimizing queues.
"""
parameters = [parameters[n] for n in self.__dimension_names]
if parameters in self.__leased_experiments:
self.__leased_experiments.remove(parameters)
do_fit_model = len(self.__opt.yi) >= self.__min_num_results_to_fit
# Unfortunate hack: this depends on the internals.
if do_fit_model:
self.__opt._n_random_starts = 0 # Since we have adequately many results, stop using random
self.__opt.tell(parameters, result, fit=do_fit_model)
def __load_dimensions(self, dimensions_file:str)->Dict:
with open(dimensions_file) as f:
dimensions = json.load(f)
def parse_dimension(specs: Dict[str, Any]):
if specs['type'] == 'Real':
return specs['name'], Real(specs['low'], specs['high'])
elif specs['type'] == 'Integer':
return specs['name'], Integer(specs['low'], specs['high'])
elif specs['type'] == 'Categorical':
return specs['name'], Categorical(specs['categories'])
else:
raise Exception('Unrecognized dimension type %s' % specs['type'])
return OrderedDict([parse_dimension(d) for d in dimensions])
def init_opt(self, bounds):
return Optimizer(dimensions=bounds,
random_state=1,
base_estimator='gp',
n_initial_points=2 * self.n_points)
def main_rule_utils(config, main_loop_seed=MAIN_LOOP_INITIAL_SEED):
rule_utils.store_config_on_first_run(config)
experience_buffer = utils.ExperienceBuffer(config['max_experience_buffer'])
config_keys = list(config['initial_config_ranges'].keys())
if deterministic_games:
utils.set_seed(main_loop_seed)
fixed_pool_mode = config['play_fixed_pool_only']
if fixed_pool_mode:
fixed_opp_repeats = config['fixed_opponents_num_repeat_first_configs']
config_has_range = isinstance(
list(config['initial_config_ranges'].values())[0], tuple)
# Prepare the Bayesian optimizer
if config_has_range:
opt_range = [config['initial_config_ranges'][k][0] for k in config_keys]
opt = Optimizer(opt_range)
if config['play_fixed_pool_fit_prev_data']:
fixed_pool_experience_path = rule_utils.get_self_play_experience_path(
config['pool_name'])
if os.path.exists(fixed_pool_experience_path):
print('\nBayesian fit to earlier experiments')
this_folder = os.path.dirname(__file__)
agents_folder = os.path.join(
this_folder, '../Rule agents/' + config['pool_name'])
config_settings_path = os.path.join(
agents_folder, CONFIG_SETTINGS_EXTENSION)
if os.path.exists(config_settings_path):
config_results = pd.read_csv(config_settings_path)
suggested = config_results.iloc[:, :-1].values.tolist()
target_scores = (-config_results.iloc[:, -1].values).tolist()
opt.tell(suggested, target_scores)
# import pdb; pdb.set_trace()
# print(opt.get_result().x, opt.get_result().fun) # WRONG!
else:
opt = None
next_fixed_opponent_suggested = None
iteration_config_rewards = None
experience_features_rewards_path = None
while True:
if deterministic_games:
utils.set_seed(main_loop_seed)
# Section 1: play games against agents of N previous pools
if config['num_games_previous_pools'] and not fixed_pool_mode:
print('\nPlay vs other rule based agents from the last {} pools'.format(
config['max_pool_size']))
(self_play_experience, rules_config_path,
avg_reward_sp, _) = rule_experience.play_games(
pool_name=config['pool_name'],
num_games=config['num_games_previous_pools'],
max_pool_size=config['max_pool_size'],
num_agents=config['num_agents_per_game'],
exclude_current_from_opponents=False,
record_videos_new_iteration=config['record_videos_new_iteration'],
initial_config_ranges=config['initial_config_ranges'],
use_multiprocessing=config['use_multiprocessing'],
episode_steps_override=config['episode_steps_override'],
early_episode_termination=config['early_episode_termination_steps'],
)
experience_buffer.add(self_play_experience)
# Section 2: play games against agents of the previous pool
if config['num_games_evaluation'] and not fixed_pool_mode:
print('\nPlay vs previous iteration')
(evaluation_experience, rules_config_path,
avg_reward_eval, _) = rule_experience.play_games(
pool_name=config['pool_name'],
num_games=config['num_games_evaluation'],
max_pool_size=2,
num_agents=config['num_agents_per_game'],
exclude_current_from_opponents=True,
use_multiprocessing=config['use_multiprocessing'],
episode_steps_override=config['episode_steps_override'],
early_episode_termination=config['early_episode_termination_steps'],
)
# experience_buffer.add(evaluation_experience)
if fixed_pool_mode:
if iteration_config_rewards is not None:
# Update the optimizer using the most recent fixed opponent pool
# results
target_scores = np.reshape(-iteration_config_rewards[
'episode_reward'].values, [-1, fixed_opp_repeats]).mean(1).tolist()
if config_has_range:
opt.tell(next_fixed_opponent_suggested, target_scores)
# Append the tried settings to the settings-scores file
config_rewards = rule_utils.append_config_scores(
next_fixed_opponent_suggested, target_scores, config_keys,
config['pool_name'], CONFIG_SETTINGS_EXTENSION)
# Update the plot of the tried settings and obtained scores
rule_utils.plot_reward_versus_features(
experience_features_rewards_path, config_rewards,
target_col="Average win rate", include_all_targets=True,
plot_name_suffix="config setting average win rate", all_scatter=True)
# Select the next hyperparameters to try
try:
next_fixed_opponent_suggested, next_fixed_opponent_configs = (
rule_utils.get_next_config_settings(
opt, config_keys, config['num_games_fixed_opponents_pool'],
fixed_opp_repeats, config['initial_config_ranges'])
)
except:
import pdb; pdb.set_trace()
# Section 3: play games against a fixed opponents pool
if config['num_games_fixed_opponents_pool']:
print('\nPlay vs the fixed opponents pool')
(fixed_opponents_experience, rules_config_path,
avg_reward_fixed_opponents, opponent_rewards) = (
rule_experience.play_games(
pool_name=config['pool_name'],
num_games=config['num_games_fixed_opponents_pool'],
max_pool_size=1, # Any positive integer is fine
num_agents=config['num_agents_per_game'],
exclude_current_from_opponents=False,
fixed_opponent_pool=True,
initial_config_ranges=config['initial_config_ranges'],
use_multiprocessing=config['use_multiprocessing'],
num_repeat_first_configs=fixed_opp_repeats,
first_config_overrides=next_fixed_opponent_configs,
episode_steps_override=config['episode_steps_override'],
early_episode_termination=config['early_episode_termination_steps'],
)
)
experience_buffer.add(fixed_opponents_experience)
# import pdb; pdb.set_trace()
# Select the values that will be used to determine if a next iteration file
# will be created
serialized_raw_experience = fixed_opponents_experience if (
fixed_pool_mode) else self_play_experience
# Optionally append the experience of interest to disk
iteration_config_rewards = (
rule_utils.serialize_game_experience_for_learning(
serialized_raw_experience, fixed_pool_mode, config_keys))
if config['save_experience_data_to_disk']:
experience_features_rewards_path = rule_utils.write_experience_data(
config['pool_name'], iteration_config_rewards)
# Section 4: Update the iteration, store videos and record learning
# progress.
if fixed_pool_mode:
update_config = {'Time stamp': str(datetime.now())}
for i in range(len(opponent_rewards)):
update_config['Reward ' + opponent_rewards[i][2]] = np.round(
opponent_rewards[i][1]/(1e-10+opponent_rewards[i][0]), 2)
rule_utils.update_learning_progress(config['pool_name'], update_config)
config_override_agents = (
fixed_opponents_experience[0].config_game_agents)
# import pdb; pdb.set_trace()
rule_utils.record_videos(
rules_config_path, config['num_agents_per_game'],
extension_override=str(datetime.now())[:19],
config_override_agents=config_override_agents,
env_seed_deterministic=fixed_opponents_experience[0].env_random_seed,
rng_action_seeds=fixed_opponents_experience[0].act_random_seeds,
first_game_recording=fixed_opponents_experience[0].game_recording,
deterministic_games=config['deterministic_games'],
deterministic_extension=f" - Seed {main_loop_seed}")
else:
# Save a new iteration if it has significantly improved
data_rules_path = rules_config_path
if min(avg_reward_sp, avg_reward_eval) >= config[
'min_new_iteration_win_rate']:
original_rules_config_path = rules_config_path
incremented_rules_path = utils.increment_iteration_id(
rules_config_path, extension='.json')
copyfile(rules_config_path, incremented_rules_path)
rules_config_path = incremented_rules_path
if config['record_videos_new_iteration']:
rule_utils.record_videos(
original_rules_config_path, config['num_agents_per_game'])
elif config['record_videos_each_main_loop']:
rule_utils.record_videos(
rules_config_path, config['num_agents_per_game'],
str(datetime.now())[:19])
# Record learning progress
# import pdb; pdb.set_trace()
rule_utils.update_learning_progress(config['pool_name'], {
'Time stamp': str(datetime.now()),
'Average reward self play': avg_reward_sp,
'Average evaluation reward': avg_reward_eval,
'Experience buffer size': experience_buffer.size(),
'Data rules path': data_rules_path,
})
# Section 5: Update the latest config range using the data in the
# experience buffer
if rules_config_path is not None:
if not fixed_pool_mode:
# Evolve the config ranges in a very simple gradient free way.
rule_utils.evolve_config(
rules_config_path, iteration_config_rewards,
config['initial_config_ranges'])
# Create plot(s) of the terminal reward as a function of all serialized
# features
if config['save_experience_data_to_disk']:
rule_utils.plot_reward_versus_features(
experience_features_rewards_path, iteration_config_rewards,
plot_name_suffix=str(datetime.now())[:19])
main_loop_seed += 1
class SkOptOptimizer(PhotonBaseOptimizer):
def __init__(
self,
n_configurations: int = 20,
acq_func: str = "gp_hedge",
acq_func_kwargs: dict = None,
):
self.optimizer = None
self.hyperparameter_list = []
self.metric_to_optimize = ""
self.ask = self.ask_generator()
self.n_configurations = n_configurations
self.acq_func = acq_func
self.acq_func_kwargs = acq_func_kwargs
self.maximize_metric = True
self.constant_dictionary = {}
def prepare(self, pipeline_elements: list, maximize_metric: bool):
self.hyperparameter_list = []
self.maximize_metric = maximize_metric
# build space
space = []
for pipe_element in pipeline_elements:
if hasattr(pipe_element, "hyperparameters"):
for name, value in pipe_element.hyperparameters.items():
# if we only have one value we do not need to optimize
if isinstance(value, list) and len(value) < 2:
self.constant_dictionary[name] = value[0]
continue
if isinstance(value,
PhotonCategorical) and len(value.values) < 2:
self.constant_dictionary[name] = value.values[0]
continue
skopt_param = self._convert_PHOTON_to_skopt_space(
value, name)
if skopt_param is not None:
space.append(skopt_param)
if len(space) == 0:
logger.warn(
"Did not find any hyperparameters to convert into skopt space")
self.optimizer = None
else:
self.optimizer = Optimizer(
space,
"ET",
acq_func=self.acq_func,
acq_func_kwargs=self.acq_func_kwargs,
)
self.ask = self.ask_generator()
def _convert_PHOTON_to_skopt_space(self, hyperparam: object, name: str):
if not hyperparam:
return None
self.hyperparameter_list.append(name)
if isinstance(hyperparam, PhotonCategorical):
return skoptCategorical(hyperparam.values, name=name)
elif isinstance(hyperparam, list):
return skoptCategorical(hyperparam, name=name)
elif isinstance(hyperparam, FloatRange):
if hyperparam.range_type == "linspace":
return Real(hyperparam.start,
hyperparam.stop,
name=name,
prior="uniform")
elif hyperparam.range_type == "logspace":
return Real(hyperparam.start,
hyperparam.stop,
name=name,
prior="log-uniform")
else:
return Real(hyperparam.start, hyperparam.stop, name=name)
elif isinstance(hyperparam, IntegerRange):
return Integer(hyperparam.start, hyperparam.stop, name=name)
def ask_generator(self):
if self.optimizer is None:
yield {}
else:
for i in range(self.n_configurations):
next_config_list = self.optimizer.ask()
next_config_dict = {
self.hyperparameter_list[number]:
self._convert_to_native(value)
for number, value in enumerate(next_config_list)
}
yield next_config_dict
def _convert_to_native(self, obj):
# check if we have a numpy object, if so convert it to python native
if type(obj).__module__ == np.__name__:
return np.asscalar(obj)
else:
return obj
def tell(self, config, performance):
# convert dictionary to list in correct order
if self.optimizer is not None:
config_values = [config[name] for name in self.hyperparameter_list]
best_config_metric_performance = performance[1]
if self.maximize_metric:
if isinstance(best_config_metric_performance, list):
print("BEST CONFIG METRIC PERFORMANCE: " +
str(best_config_metric_performance))
best_config_metric_performance = best_config_metric_performance[
0]
best_config_metric_performance = -best_config_metric_performance
# random_accuracy = np.random.randn(1)[0]
self.optimizer.tell(config_values, best_config_metric_performance)
def plot_evaluations(self):
results = SkoptResults()
results.space = self.optimizer.space
results.x_iters = self.optimizer.Xi
results = self._convert_categorical_hyperparameters(results)
results.x = results.x_iters[np.argmin(self.optimizer.yi)]
plt.figure(figsize=(10, 10))
return plot_evaluations(results)
def plot_objective(self):
results = SkoptResults()
results.space = self.optimizer.space
results.x_iters = self.optimizer.Xi
results = self._convert_categorical_hyperparameters(results)
results.x = results.x_iters[np.argmin(self.optimizer.yi)]
results.models = self.optimizer.models
plt.figure(figsize=(10, 10))
return plot_objective(results)
def _convert_categorical_hyperparameters(self, results):
parameter_types = list()
for i, dim in enumerate(results.space.dimensions):
if isinstance(dim, skoptCategorical):
parameter_types.append(dim.transformer)
setattr(results.space.dimensions[i], "categories",
dim.transformed_bounds)
else:
parameter_types.append(False)
for i, xs in enumerate(results.x_iters):
for k, xsk in enumerate(xs):
if parameter_types[k]:
results.x_iters[i][k] = parameter_types[k].transform([xsk])
return results
func=use_func,
dimensions=dim,
n_calls = run_for,
)
print "GPM best value",res.fun,"at",res.x
#print res
print "took",time.mktime(time.gmtime())-start,"[s]"
o = Optimizer(
n_initial_points =5,
acq_func = 'gp_hedge',
acq_optimizer='auto',
base_estimator=GaussianProcessRegressor(alpha=0.0, copy_X_train=True,
n_restarts_optimizer=2,
noise='gaussian', normalize_y=True,
optimizer='fmin_l_bfgs_b'),
dimensions=dim,
)
m = manager(n = 4,
skobj = o,
iterations = run_for,
func = use_func,
wait= 0
)
start = time.mktime(time.gmtime())
m.run()
import numpy as np
best = np.argmin( m.sk.yi)
def init_optimization(optim_config_file, ctx):
with optim_config_file.open('r') as f:
optim_config = yaml.load(f, Loader=yaml.SafeLoader)
# default settings
if "objective" not in optim_config:
raise ValueError(
'ERROR: the configuration must provide an `objective`.')
if "evaluations" not in optim_config:
raise ValueError(
'ERROR: the configuration must project a `evaluations` budget.')
optim_config = {
"solver": {},
"search_space": {},
"preset_params": {},
**optim_config,
}
optim_config['solver'] = {
"name": "scikit-optimize",
"random_state": 42,
**optim_config.get('solver', {}),
}
from skopt.utils import Space
space = Space.from_yaml(optim_config_file, namespace='search_space')
preset_params = optim_config.get('preset_params', {})
click.secho("Search space:", fg="blue", err=True)
click.secho(str(space), fg="blue", dim=True, err=True)
click.secho("Preset parameters:", fg="blue", err=True)
click.secho(str(preset_params), fg="blue", dim=True, err=True)
# we use the iteration step in the objective function, to store results at the right place
from skopt.utils import Integer
dim_iteration = Integer(name='iteration', low=0, high=2 ^ 16)
dims = [*space, dim_iteration]
from skopt.utils import use_named_args
@use_named_args(dims)
def objective(**opt_params):
params = {**preset_params, **opt_params}
# From the UI we will want to see the iteration as a metric
del params["iteration"]
batch_label = f"{ctx.obj['batch_label']}|iter{opt_params['iteration']+1}"
command = ' '.join([
'qa',
f"--label '{batch_label}'",
f"--platform '{ctx.obj['platform']}'",
f"--configuration '{ctx.obj['configuration']}'",
f"--tuning '{json.dumps(params, sort_keys=True, cls=NumpyEncoder)}'",
'batch',
' '.join(
[f'--batches-file "{b}"' for b in ctx.obj["batches_files"]]),
' '.join([f'--batch "{b}"' for b in ctx.obj["batches"]]),
# we notably forward --batch
' '.join(ctx.obj["forwarded_args"]),
])
click.secho(command, fg="blue")
import re
command = re.sub(
'^qa', 'python -m qaboard',
command) # helps make sure we run the right thing when testing
if str(subproject) != '.':
command = f"cd {subproject} && {command}"
if not ctx.obj['dryrun']:
p = subprocess.run(
command,
shell=True,
encoding="utf-8",
)
if p.returncode != 0:
click.secho(
f'[ERROR ({p.returncode})] Check the logs in QA-Board to know what output failed',
fg='red',
bold=True)
# Now that we finished computing all the results, we will download the results and
# compute the objective function:
return batch_objective(batch_label, optim_config['objective'])
# For the full list of options, refer to:
# https://scikit-optimize.github.io/stable/modules/generated/skopt.optimizer.Optimizer.html#skopt.optimizer.Optimizer
from skopt import Optimizer
del optim_config['solver']['name']
optimizer = Optimizer(space, **optim_config['solver'])
# in the optimization loop, `ask` gives us an array of values
# this wrapper converts it back to the actual named parameters
@use_named_args([*space])
def dim_mapping(**opt_params):
return {**preset_params, **opt_params}
return objective, optimizer, optim_config, dim_mapping
def run():
start_time = time.time()
print("run() start: {}".format(str(datetime.datetime.now())))
comm = MPI.COMM_WORLD # get MPI communicator object
size = comm.size # total number of processes
rank = comm.rank # rank of this process
status = MPI.Status() # get MPI status object
print("ME rank is {}".format(rank))
instance = problem.Problem()
spaceDict = instance.space
params = instance.params
global problem_params
problem_params = params
starting_point = instance.starting_point
# handshake to ensure working
eqpy.OUT_put("Params")
# initial parameter set telling us the number of times to run the loop
initparams = eqpy.IN_get()
(init_size, max_evals, num_workers, num_buffer, seed, max_threshold,
n_jobs) = eval('{}'.format(initparams))
space = [spaceDict[key] for key in params]
print(space)
parDict = {}
resultsList = []
parDict['kappa'] = 1.96
# can set to num cores
parDict['n_jobs'] = n_jobs
init_x = []
opt = Optimizer(space,
base_estimator='RF',
acq_optimizer='sampling',
acq_func='LCB',
acq_func_kwargs=parDict,
random_state=seed)
eval_counter = 0
askedDict = {}
print(
"Master starting with {} init_size, {} max_evals, {} num_workers, {} num_buffer, {} max_threshold"
.format(init_size, max_evals, num_workers, num_buffer, max_threshold))
x = opt.ask(n_points=init_size)
res, resstring = create_list_of_json_strings(x)
print("Initial design is {}".format(resstring))
for r, xx in zip(res, x):
askedDict[r] = xx
eqpy.OUT_put(resstring)
currently_out = init_size
total_out = init_size
results = []
group = comm.Get_group()
# Assumes only one adlb_server
# num_workers + 1 = num_turbine_workers
newgroup = group.Excl([num_workers + 1])
#print("ME newgroup size is {}".format(newgroup.size))
newcomm = comm.Create_group(newgroup, 1)
nrank = newcomm.rank
#print("ME nrank is {}".format(nrank))
counter_threshold = 1
counter = 0
end_iter_time = 0
while eval_counter < max_evals:
start_iter_time = time.time()
print("\neval_counter = {}".format(eval_counter))
data = newcomm.recv(source=MPI.ANY_SOURCE, status=status)
counter = counter + 1
xstring = data['x']
x = askedDict[xstring]
y = data['cost']
if math.isnan(y):
y = sys.float_info.max
opt.tell(x, y)
#source = status.Get_source()
#tag = status.Get_tag()
elapsed_time = float(time.time() - start_time)
print('elapsed_time:%1.3f' % elapsed_time)
results.append(str(data))
eval_counter = eval_counter + 1
currently_out = currently_out - 1
# if jobs are finishing within 16 seconds of
# each other, then batch the point production
if start_iter_time - end_iter_time < 16:
counter_threshold = max_threshold
if max_evals - eval_counter < counter_threshold:
counter_threshold = max_evals - eval_counter
if counter_threshold > currently_out:
counter_threshold = currently_out
else:
counter_threshold = 1
print("counter_threshold: {}".format(counter_threshold))
print("currently_out:{}, total_out:{}".format(currently_out,
total_out))
if currently_out < num_workers + num_buffer and total_out < max_evals and counter >= counter_threshold:
n_points = counter
if n_points + total_out > max_evals:
n_points = max_evals - total_out
ts = time.time()
x = opt.ask(n_points=n_points)
res, resstring = create_list_of_json_strings(x)
for r, xx in zip(res, x):
askedDict[r] = xx
eqpy.OUT_put(resstring)
print('point production elapsed_time:%1.3f' %
float(time.time() - ts))
currently_out = currently_out + n_points
total_out = total_out + n_points
counter = 0
end_iter_time = start_iter_time
print('Search finishing')
eqpy.OUT_put("DONE")
eqpy.OUT_put(";".join(results))
try:
with open('my_optimizer_old.pkl', 'rb') as f:
opt = pickle.load(f)
except:
# opt = Optimizer([(-2.0, 2.0)], "GP", acq_func = 'LCB', acq_func_kwargs = )
#### high k means explore, low kappa means exploit
# bounds = [(0., 1.),] * 8
opt = Optimizer(
base_estimator='GP',
acq_func='LCB',
acq_optimizer='auto',
dimensions=[
(-5.0, 5.0), # range for param 1 (eg trajectory final height?)
(-5.0, 5.0), # range for param 2 (eg trajectory final pitch?)
(-5.0, 5.0)
], # range for param 4 (eg vibrational state duration?)
acq_func_kwargs={
'kappa': 10
}, # we should prefer explore (high kappa). howver, with higher dim it will naturally tend to diversify, so kappa could be decreased
n_initial_points=10)
"""
kappa [float, default=1.96]: Controls how much of the variance in the predicted values should be taken into account. Used when the acquisition is "LCB" (lower confidence bound).
If set high, then we are favouring exploration over exploitation and vice versa.
xi [float, default=0.01]: Controls how much improvement one wants over the previous best values. Used when the acquisition is either "EI" or "PI".
to use this, i think i need a way to scale xi by the variance of the signal, which howver will depend on the participant and will have to be adapted online...
"""
help="Specify the out csv filename.",
required=True)
args = parser.parse_args()
# Defining the hyperspace
hyperparameters = [
(0.00001, 0.1), # learning_rate
(0.2, 0.9), # dropout
(10, 100), # epochs
(10, 1000)
] # batch size
space = create_hyperspace(hyperparameters)
# Perform runs and aggregate results
results = []
for section in tqdm(space):
# create a skopt gp minimize object
optimizer = Optimizer(section)
search_algo = SkOptSearch(
optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'],
metric='average_res',
mode='max')
analysis = tune.run(multi_train,
search_alg=search_algo,
num_samples=50,
resources_per_trial={'gpu': 1})
results.append(analysis)
all_pt_results = results[0].results_df
for i in range(1, len(results)):
all_pt_results = all_pt_results.append(results[i].results_df)
all_pt_results.to_csv(args.out)
def fit(self,
X,
Y,
total_duration=6e7,
n_iter=100,
cv_iter=None,
optimizer=None,
acq_func='gp_hedge',
**kwargs):
start = datetime.now()
def splitter(itr):
for train_idx, test_idx in itr:
yield X[train_idx], Y[train_idx], X[test_idx], Y[test_idx]
def splitter_dict(itr_dict):
n_splits = len(list(itr_dict.values())[0])
for i in range(n_splits):
X_train = dict()
Y_train = dict()
X_test = dict()
Y_test = dict()
for n_obj, itr in itr_dict.items():
train_idx = itr[i][0]
test_idx = itr[i][1]
X_train[n_obj] = np.copy(X[n_obj][train_idx])
X_test[n_obj] = np.copy(X[n_obj][test_idx])
Y_train[n_obj] = np.copy(Y[n_obj][train_idx])
Y_test[n_obj] = np.copy(Y[n_obj][test_idx])
yield X_train, Y_train, X_test, Y_test
if cv_iter is None:
cv_iter = ShuffleSplit(n_splits=3,
test_size=0.1,
random_state=self.random_state)
if isinstance(X, dict):
splits = dict()
for n_obj, arr in X.items():
if arr.shape[0] == 1:
splits[n_obj] = [([0], [0])
for i in range(cv_iter.n_splits)]
else:
splits[n_obj] = list(cv_iter.split(arr))
else:
splits = list(cv_iter.split(X))
# Pre-compute splits for reuse
# Here we fix a random seed for all simulations to correlate the random
# streams:
seed = self.random_state.randint(2**32, dtype='uint32')
self.logger.debug(
'Random seed for the ranking algorithm: {}'.format(seed))
opt_seed = self.random_state.randint(2**32, dtype='uint32')
self.logger.debug('Random seed for the optimizer: {}'.format(opt_seed))
gp_seed = self.random_state.randint(2**32, dtype='uint32')
self.logger.debug(
'Random seed for the GP surrogate: {}'.format(gp_seed))
if optimizer is not None:
opt = optimizer
self.logger.debug('Setting the provided optimizer')
self.log_best_params(opt)
else:
transformed = []
for param in self.parameter_ranges:
transformed.append(check_dimension(param))
self.logger.info("Parameter Space: {}".format(transformed))
space = normalize_dimensions(transformed)
self.logger.info(
"Parameter Space after transformation: {}".format(space))
# Todo: Make this passable
base_estimator = cook_estimator("GP",
space=space,
random_state=gp_seed,
noise="gaussian")
opt = Optimizer(dimensions=self.parameter_ranges,
random_state=opt_seed,
base_estimator=base_estimator,
acq_func=acq_func,
**kwargs)
self._callbacks_set_optimizer(opt)
self._callbacks_on_optimization_begin()
time_taken = duration_tillnow(start)
total_duration -= time_taken
max_fit_duration = -10000
self.logger.info('Time left for {} iterations is {}'.format(
n_iter, microsec_to_time(total_duration)))
try:
for t in range(n_iter):
start = datetime.now()
self._callbacks_on_iteration_begin(t)
self.logger.info(
'Starting optimization iteration: {}'.format(t))
if t > 0:
self.log_best_params(opt)
next_point = opt.ask()
self.logger.info('Next parameters:\n{}'.format(next_point))
results = []
running_times = []
if isinstance(X, dict):
for X_train, Y_train, X_test, Y_test in splitter_dict(
splits):
result, time_taken = self._fit_ranker(
X_train, Y_train, X_test, Y_test, next_point)
running_times.append(time_taken)
results.append(result)
else:
for X_train, Y_train, X_test, Y_test in splitter(splits):
result, time_taken = self._fit_ranker(
X_train, Y_train, X_test, Y_test, next_point)
running_times.append(time_taken)
results.append(result)
results = np.array(results)
running_times = np.array(running_times)
mean_result = np.mean(results)
mean_fitting_duration = np.mean(running_times)
# Storing the maximum time to run the splitting model and adding the time for out of sample evaluation
if max_fit_duration < np.sum(running_times):
max_fit_duration = np.sum(running_times)
self.logger.info(
'Validation error for the parameters is {:.4f}'.format(
mean_result))
self.logger.info('Time taken for the parameters is {}'.format(
microsec_to_time(np.sum(running_times))))
if "ps" in opt.acq_func:
opt.tell(next_point, [mean_result, mean_fitting_duration])
else:
opt.tell(next_point, mean_result)
self._callbacks_on_iteration_end(t)
self.logger.info(
"Main optimizer iterations done {} and saving the model".
format(np.array(opt.yi).shape[0]))
dump(opt, self.optimizer_path)
time_taken = duration_tillnow(start)
total_duration -= time_taken
self.logger.info('Time left for simulations is {} '.format(
microsec_to_time(total_duration)))
if (total_duration - max_fit_duration) < 0:
self.logger.info(
'At iteration {} maximum time required by model to validate a parameter values'
.format(microsec_to_time(max_fit_duration)))
self.logger.info(
'At iteration {} simulation stops, due to time deficiency'
.format(t))
break
except KeyboardInterrupt:
self.logger.debug(
'Optimizer interrupted saving the model at {}'.format(
self.optimizer_path))
self.log_best_params(opt)
else:
self.logger.debug(
'Finally, fit a model on the complete training set and storing the model at {}'
.format(self.optimizer_path))
self._fit_params["epochs"] = self._fit_params.get("epochs", 1000)
if "ps" in opt.acq_func:
best_point = opt.Xi[np.argmin(np.array(opt.yi)[:, 0])]
else:
best_point = opt.Xi[np.argmin(opt.yi)]
self._set_new_parameters(best_point)
self.model = copy.copy(self.ranker)
self.model.fit(X, Y, **self._fit_params)
finally:
self._callbacks_on_optimization_end()
self.optimizer = opt
if np.array(opt.yi).shape[0] != 0:
dump(opt, self.optimizer_path)
def test_n_random_starts_Optimizer():
# n_random_starts got renamed in v0.4
et = ExtraTreesRegressor(random_state=2)
with pytest.deprecated_call():
Optimizer([(0, 1.)], et, n_random_starts=10, acq_optimizer='sampling')
class Optimizer:
SEED = 12345
def __init__(
self,
problem,
num_workers,
surrogate_model="RF",
acq_func="gp_hedge",
acq_kappa=1.96,
liar_strategy="cl_max",
n_jobs=1,
**kwargs,
):
assert surrogate_model in [
"RF",
"ET",
"GBRT",
"GP",
"DUMMY",
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.space = problem.space
# queue of remaining starting points
self.starting_points = problem.starting_point
n_init = (inf if surrogate_model == "DUMMY" else max(
num_workers, len(self.starting_points)))
self._optimizer = SkOptimizer(
dimensions=self.space,
base_estimator=base_estimator,
acq_optimizer="sampling",
acq_func=acq_func,
acq_func_kwargs={"kappa": acq_kappa},
random_state=self.SEED,
n_initial_points=n_init,
)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info(
f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def _get_lie(self):
if self.strategy == "cl_min":
return min(self._optimizer.yi) if self._optimizer.yi else 0.0
elif self.strategy == "cl_mean":
return np.mean(self._optimizer.yi) if self._optimizer.yi else 0.0
else:
return max(self._optimizer.yi) if self._optimizer.yi else 0.0
def _xy_from_dict(self):
XX = []
for key in self.evals.keys():
x = tuple(convert2np(k) for k in key)
XX.append(x)
YY = [-self.evals[x] for x in self.evals.keys()] # ! "-" maximizing
return XX, YY
def dict_to_xy(self, xy_dict: dict):
XX = []
for key in xy_dict.keys():
x = [convert2np(k) for k in key]
XX.append(x)
YY = [-xy_dict[x] for x in xy_dict.keys()] # ! "-" maximizing
return XX, YY
def to_dict(self, x: list) -> dict:
res = {}
hps_names = self.space.get_hyperparameter_names()
for i in range(len(x)):
res[hps_names[i]] = "nan" if isnan(x[i]) else x[i]
return res
def _ask(self):
if len(self.starting_points) > 0:
x = self.starting_points.pop()
else:
x = self._optimizer.ask()
y = self._get_lie()
key = tuple(self.to_dict(x).values())
if key not in self.evals:
self.counter += 1
self._optimizer.tell(x, y)
self.evals[key] = y
logger.debug(f"_ask: {x} lie: {y}")
else:
logger.debug(f"Duplicate _ask: {x} lie: {y}")
return self.to_dict(x)
def ask(self, n_points=None, batch_size=20):
if n_points is None:
return self._ask()
else:
batch = []
for _ in range(n_points):
batch.append(self._ask())
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
def ask_initial(self, n_points):
if len(self.starting_points) > 0:
XX = [
self.starting_points.pop()
for i in range(min(n_points, len(self.starting_points)))
]
if len(XX) < n_points:
XX += self._optimizer.ask(n_points=n_points - len(XX))
else:
XX = self._optimizer.ask(n_points=n_points)
for x in XX:
y = self._get_lie()
x = [convert2np(xi) for xi in x]
key = tuple(self.to_dict(x).values())
if key not in self.evals:
self.counter += 1
self._optimizer.tell(x, y)
self.evals[key] = y
return [self.to_dict(x) for x in XX]
def tell(self, xy_data):
assert isinstance(xy_data,
list), f"where type(xy_data)=={type(xy_data)}"
minval = min(self._optimizer.yi) if self._optimizer.yi else 0.0
xy_dict = {}
for x, y in xy_data:
key = tuple(x[k] for k in self.space)
assert key in self.evals, f"where key=={key} and self.evals=={self.evals}"
logger.debug(f"tell: {x} --> {key}: evaluated objective: {y}")
self.evals[key] = y if y > np.finfo(np.float32).min else minval
xy_dict[key] = y if y > np.finfo(np.float32).min else minval
XX, YY = self.dict_to_xy(xy_dict)
selection = [
(xi, yi) for xi, yi in zip(self._optimizer.Xi, self._optimizer.yi)
if not (any([equal_list(xi, x)
for x in XX])) # all([diff(xi, x) for x in XX])
]
new_Xi, new_yi = list(zip(*selection)) if len(selection) > 0 else ([],
[])
new_Xi, new_yi = list(new_Xi), list(new_yi)
self._optimizer.Xi = new_Xi
self._optimizer.yi = new_yi
self._optimizer.tell(XX, YY)
assert len(self._optimizer.Xi) == len(
self._optimizer.yi) == self.counter, (
f"where len(self._optimizer.Xi)=={len(self._optimizer.Xi)}, "
f"len(self._optimizer.yi)=={len(self._optimizer.yi)},"
f"self.counter=={self.counter}")
class SkOptOptimizer(PhotonSlaveOptimizer):
def __init__(self,
n_configurations: int = 20,
acq_func: str = 'gp_hedge',
acq_func_kwargs: dict = None):
self.optimizer = None
self.hyperparameter_list = []
self.metric_to_optimize = ''
self.ask = self.ask_generator()
self.n_configurations = n_configurations
self.acq_func = acq_func
self.acq_func_kwargs = acq_func_kwargs
self.maximize_metric = True
self.constant_dictionary = {}
def prepare(self, pipeline_elements: list, maximize_metric: bool):
self.hyperparameter_list = []
self.maximize_metric = maximize_metric
# build space
space = []
for pipe_element in pipeline_elements:
if pipe_element.__class__.__name__ == 'Switch':
error_msg = 'Scikit-Optimize cannot operate in the specified hyperparameter space with a Switch ' \
'element. We recommend the use of SMAC.'
logger.error(error_msg)
raise ValueError(error_msg)
if hasattr(pipe_element, 'hyperparameters'):
for name, value in pipe_element.hyperparameters.items():
# if we only have one value we do not need to optimize
if isinstance(value, list) and len(value) < 2:
self.constant_dictionary[name] = value[0]
continue
if isinstance(value,
PhotonCategorical) and len(value.values) < 2:
self.constant_dictionary[name] = value.values[0]
continue
skopt_param = self._convert_PHOTON_to_skopt_space(
value, name)
if skopt_param is not None:
space.append(skopt_param)
if len(space) == 0:
logger.warning(
"Did not find any hyperparameters to convert into skopt space")
self.optimizer = None
else:
self.optimizer = Optimizer(space,
"ET",
acq_func=self.acq_func,
acq_func_kwargs=self.acq_func_kwargs)
self.ask = self.ask_generator()
def _convert_PHOTON_to_skopt_space(self, hyperparam: object, name: str):
if not hyperparam:
return None
self.hyperparameter_list.append(name)
if isinstance(hyperparam, PhotonCategorical):
return skoptCategorical(hyperparam.values, name=name)
elif isinstance(hyperparam, list):
return skoptCategorical(hyperparam, name=name)
elif isinstance(hyperparam, FloatRange):
if hyperparam.range_type == 'linspace':
return Real(hyperparam.start,
hyperparam.stop,
name=name,
prior='uniform')
elif hyperparam.range_type == 'logspace':
return Real(hyperparam.start,
hyperparam.stop,
name=name,
prior='log-uniform')
else:
return Real(hyperparam.start, hyperparam.stop, name=name)
elif isinstance(hyperparam, IntegerRange):
return Integer(hyperparam.start, hyperparam.stop, name=name)
def ask_generator(self):
if self.optimizer is None:
yield {}
else:
for i in range(self.n_configurations):
next_config_list = self.optimizer.ask()
next_config_dict = {
self.hyperparameter_list[number]:
self._convert_to_native(value)
for number, value in enumerate(next_config_list)
}
yield next_config_dict
def _convert_to_native(self, obj):
# check if we have a numpy object, if so convert it to python native
if type(obj).__module__ == np.__name__:
return obj.item()
else:
return obj
def tell(self, config, performance):
# convert dictionary to list in correct order
if self.optimizer is not None:
config_values = [config[name] for name in self.hyperparameter_list]
best_config_metric_performance = performance[1]
if self.maximize_metric:
best_config_metric_performance = -best_config_metric_performance
# random_accuracy = np.random.randn(1)[0]
self.optimizer.tell(config_values, best_config_metric_performance)
