def run(self):
import skopt
optimizer = self.input().load(
) if self.branch != 0 else skopt.Optimizer(
dimensions=[
skopt.space.Real(-5.0, 10.0),
skopt.space.Real(0.0, 15.0)
],
random_state=1,
n_initial_points=self.n_initial_points)
x = optimizer.ask(n_points=self.n_parallel)
output = yield Objective.req(self,
x=x,
iteration=self.branch,
branch=-1)
y = [f.load()["y"] for f in output["collection"].targets.values()]
optimizer.tell(x, y)
print("minimum after {} iterations: {}".format(self.branch + 1,
min(optimizer.yi)))
with self.output().localize("w") as tmp:
tmp.dump(optimizer)
def __init__(self, comm, num_blocks,
opt_params, ga, populationSize):
print("Coordinator initializing")
self.comm = comm
self.num_blocks = num_blocks
self.opt_params = opt_params
self.ga = ga
if ga:
self.populationSize = populationSize
self.optimizer = GA(self.opt_params, self.populationSize)
else:
self.optimizer = skopt.Optimizer(dimensions=self.opt_params, random_state= 13579)
self.param_list = []
self.fom_list = []
self.block_dict = {}
self.req_dict = {}
self.best_params = None
self.best_fom = None
self.next_params = []
self.to_tell = []
self.ends_cycle = False
self.target_fom = None
self.history = {}
self.label = None
self.iter = 1
def create_optimizer(self):
skopt_search_space = []
for param in self.search_space.params:
if param.type == INTEGER:
skopt_search_space.append(
skopt.space.Integer(int(param.min),
int(param.max),
name=param.name))
elif param.type == DOUBLE:
skopt_search_space.append(
skopt.space.Real(float(param.min),
float(param.max),
"log-uniform",
name=param.name))
elif param.type == CATEGORICAL or param.type == DISCRETE:
skopt_search_space.append(
skopt.space.Categorical(param.list, name=param.name))
self.skopt_optimizer = skopt.Optimizer(
skopt_search_space,
base_estimator=self.base_estimator,
n_initial_points=self.n_initial_points,
acq_func=self.acq_func,
acq_optimizer=self.acq_optimizer,
random_state=self.random_state)
def _setup_skopt(self):
_validate_warmstart(self._parameter_names, self._points_to_evaluate,
self._evaluated_rewards)
if not self._skopt_opt:
if not self._space:
raise ValueError(
"If you don't pass an optimizer instance to SkOptSearch, "
"pass a valid `space` parameter.")
self._skopt_opt = sko.Optimizer(self._parameter_ranges)
if self._points_to_evaluate and self._evaluated_rewards:
self._skopt_opt.tell(self._points_to_evaluate,
self._evaluated_rewards)
elif self._points_to_evaluate:
self._initial_points = self._points_to_evaluate
self._parameters = self._parameter_names
# Skopt internally minimizes, so "max" => -1
if self._mode == "max":
self._metric_op = -1.
elif self._mode == "min":
self._metric_op = 1.
if self._metric is None and self._mode:
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
def summarize(opt, steps):
print('Summarizing best values')
for kappa in [0] + list(np.logspace(-1, 1, steps - 1)):
new_opt = skopt.Optimizer(opt.space.dimensions,
acq_func='LCB',
acq_func_kwargs=dict(
kappa=kappa / 5,
n_jobs=-1,
),
acq_optimizer='lbfgs',
acq_optimizer_kwargs=dict(
n_restarts_optimizer=100, ))
new_opt.tell(opt.Xi, opt.yi)
x = new_opt.ask()
y, sigma = new_opt.models[-1].predict([x], return_std=True)
y = -y # Change back from minimization to maximization
def score_to_elo(score):
if score <= -1:
return float('inf')
if score >= 1:
return -float('inf')
return 400 * math.log10((1 + score) / (1 - score))
elo = score_to_elo(y)
pm = max(abs(score_to_elo(y + sigma) - elo),
abs(score_to_elo(y - sigma) - elo))
print(f'Best expectation (κ={kappa:.1f}): {x}'
f' = {y[0]:.3f} ± {sigma[0]:.3f}'
f' (ELO-diff {elo:.3f} ± {pm:.3f})')
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_max",
n_jobs=32,
**kwargs,
):
kwargs["cache_key"] = "to_dict"
super().__init__(problem, run, evaluator, **kwargs)
dhlogger.info("Initializing AMBS")
dhlogger.info(f"kappa={kappa}, xi={xi}")
self.n_initial_points = self.evaluator.num_workers
self.opt = skopt.Optimizer(
dimensions=self.problem.space,
base_estimator=self.get_surrogate_model(surrogate_model, n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={
"xi": xi,
"kappa": kappa
},
n_initial_points=self.n_initial_points,
random_state=self.problem.seed,
# model_queue_size=100,
)
def plot_convergence(self, filename=None, n_points=25, n_samples=250):
# if `savefig(filename)`, don't `show()`
if filename is not None:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import skopt
from skopt.plots import plot_convergence
spaces = list(self.spaces.values())
pnames = list(self.spaces.keys())
opt = skopt.Optimizer(spaces, "ET", acq_optimizer="sampling")
hyperpars, objective = self.trials_and_results
# skopt minimizes. Therefore use: acc = 1-acc
objective = 1-objective
hyperpars = [list(f) for f in hyperpars]
objective = list(objective)
opt.tell(hyperpars, objective)
opt_result = opt.run(lambda x: 0, n_iter=0)
plot_convergence(opt_result)#, n_points=n_points, n_samples=n_samples, dimensions=pnames)
plt.tight_layout()
if filename is None:
plt.show()
else:
plt.savefig(filename)
def __init__(self, comm, num_blocks, opt_params, ga, populationSize,
checkpointing, label):
set_logging_prefix(MPI.COMM_WORLD.Get_rank(), process_type='C')
logging.debug("Coordinator initializing")
self.comm = comm
self.num_blocks = num_blocks
self.opt_params = opt_params
self.ga = ga
if ga:
self.populationSize = populationSize
self.optimizer = GA(self.opt_params, self.populationSize)
else:
self.optimizer = skopt.Optimizer(dimensions=self.opt_params,
random_state=13579)
self.param_list = []
self.fom_list = []
self.block_dict = {}
self.req_dict = {}
self.best_params = None
self.best_fom = None
self.next_params = []
self.to_tell = []
self.ends_cycle = False
self.target_fom = None
self.history = {}
self.checkpointing = checkpointing
self.label = checkpointing if checkpointing else label
self.iter = 1
def __init__(self, search_space, skopt_kwargs):
# type: (Dict[str, BaseDistribution], Dict[str, Any]) -> None
self._search_space = search_space
dimensions = []
for name, distribution in sorted(self._search_space.items()):
# TODO(nzw0301) support IntLogUniform
if isinstance(distribution, distributions.UniformDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high)
elif isinstance(distribution, distributions.LogUniformDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high, prior="log-uniform")
elif isinstance(distribution, distributions.IntUniformDistribution):
count = (distribution.high - distribution.low) // distribution.step
dimension = space.Integer(0, count)
elif isinstance(distribution, distributions.DiscreteUniformDistribution):
count = int((distribution.high - distribution.low) // distribution.q)
dimension = space.Integer(0, count)
elif isinstance(distribution, distributions.CategoricalDistribution):
dimension = space.Categorical(distribution.choices)
else:
raise NotImplementedError(
"The distribution {} is not implemented.".format(distribution)
)
dimensions.append(dimension)
self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def doMLP():
skopt_args = collections.OrderedDict()
skopt_args["lr"] = Integer(2, 6)
skopt_args["channels"] = Integer(6, 12)
skopt_args["layers"] = Integer(1, 3)
optimizer = skopt.Optimizer(dimensions=skopt_args.values(),
base_estimator="GP",
n_initial_points=3,
random_state=args.seed)
print(skopt_args)
best_valid_metric = 0
test_for_best_valid_metric = 0
best_config = None
already_done = set()
for i in range(30):
suggestion = optimizer.ask()
if str(suggestion) in already_done:
continue
already_done.add(str(suggestion))
sdict = dict(zip(skopt_args.keys(), suggestion))
sdict["lr"] = 10**float((-sdict["lr"]))
sdict["channels"] = 2**sdict["channels"]
print(sdict)
model = models.mlp.MLP(name="MLP",
num_layer=sdict["layers"],
channels=sdict["channels"],
lr=sdict["lr"],
num_epochs=100,
patience=50,
cuda=True,
metric=sklearn.metrics.accuracy_score,
verbose=False,
seed=args.seed)
model.fit(X_train, y_train)
y_valid_pred = model.predict(X_valid)
valid_metric = sklearn.metrics.accuracy_score(
y_valid, np.argmax(y_valid_pred, axis=1))
opt_results = optimizer.tell(suggestion, -valid_metric)
#record metrics to write and plot
if best_valid_metric < valid_metric:
best_valid_metric = valid_metric
best_config = sdict
y_test_pred = model.predict(X_test)
test_metric = sklearn.metrics.accuracy_score(
y_test, np.argmax(y_test_pred, axis=1))
test_for_best_valid_metric = test_metric
print(i, "This result:", valid_metric, sdict)
print("#Final Results", test_for_best_valid_metric, best_config)
return test_metric, best_config
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
super().__init__(problem, run, evaluator, **kwargs)
dhlogger.info("Initializing AMBNAS")
dhlogger.info(
jm(
type="start_infos",
alg="ambs",
nworkers=self.evaluator.num_workers,
encoded_space=json.dumps(self.problem.space, cls=Encoder),
)
)
dhlogger.info(f"kappa={kappa}, xi={xi}")
self.n_initial_points = self.evaluator.num_workers
self.liar_strategy = liar_strategy
# Building search space for SkOptimizer using ConfigSpace
search_space = self.problem.build_search_space()
# print('ambs.py .....................', self.problem)
skopt_space = cs.ConfigurationSpace(seed=self.problem.seed)
for i, vnode in enumerate(search_space.variable_nodes):
print(f'vnode{i}.num_ops ==== {vnode.num_ops}')
hp = csh.UniformIntegerHyperparameter(
name=f"vnode_{i:05d}", lower=0, upper=(vnode.num_ops - 1)
)
skopt_space.add_hyperparameter(hp)
# dhlogger.info(f'hyliu --------------------------- {skopt_space}')
self.opt = skopt.Optimizer(
dimensions=skopt_space,
base_estimator=self.get_surrogate_model(surrogate_model, n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={"xi": xi, "kappa": kappa},
n_initial_points=self.n_initial_points,
random_state=self.problem.seed,
)
def __init__(comm, num_blocks, opt_params, model_fn):
self.comm = comm
self.num_blocks = num_blocks
self.opt_params = opt_params
self.model_fn = model_fn
self.optimizer = skopt.Optimizer(opt_params)
self.param_list = []
self.fom_list = []
self.block_dict = {}
def set_basic_conf(self):
optimizer = skopt.Optimizer([(0, 20), (-100, 100)])
previously_run_params = [[10, 0], [15, -20]]
known_rewards = [-189, -1144]
def cost(space, reporter):
reporter(loss=(space["height"]**2 + space["width"]**2))
search_alg = SkOptSearch(optimizer, ["width", "height"],
metric="loss",
mode="min",
points_to_evaluate=previously_run_params,
evaluated_rewards=known_rewards)
return search_alg, cost
def askmodel():
#Ratio Fiber/Filler
try:
fiber_w = sk.space.Real(fiber_lb, fiber_ub, name='weightratio fiber')
fillerA_w = sk.space.Real(
filler_lb, filler_ub,
name='weightratio natural filler') #if step=5, 21 options
dry_w = sk.space.Real(dry_lb, dry_ub,
name='ratio of dry material') #0.40,0.57
except ValueError:
print(
'[ASK MODEL]\t The boundaries of your model are not correctly set')
return
dim = [fiber_w, fillerA_w, dry_w]
#........... INITIALIZE MODEL ...........
opt = sk.Optimizer(dim, acq_func=acq_func)
#........... TELL MODEL HISTORY ...........
if 'fx' not in globals():
chooseoutput()
if 'fx' not in globals():
return
if 'population' not in globals():
setpop()
if 'population' not in globals():
return
global xi
#tell model history
opt.tell(x, fx)
print("[ASK MODEL]\t Succesfully build model on historical data")
#........... ASK MODEL ....................
#ask model
xi = opt.ask(population, strategy=strategy)
for i in range(population):
print(
"[ASK MODEL]\t Recipe %d/%d : fiber: %.4f | filler: %.4f | dry: %.4f"
% (i + 1, population, xi[i][0], xi[i][1], xi[i][2]))
def _init_optimizer(self, hyperparams_space, search_params):
"""
Init skopt Optimizer. It will be used to perform bayesian search.
:param hyperparams_space: user define space
:param search_params: skopt Optimizer params
"""
prepared_space = []
for param_name in hyperparams_space.keys():
prepared_space.append(
self._get_skopt_space_vals(hyperparams_space[param_name],
param_name=param_name))
optimizer = skopt.Optimizer(prepared_space, **search_params)
self._internal_optimizer = optimizer
def run(self):
import skopt
opt_params = self.optimizable_parameters()
optimizer = skopt.Optimizer(
dimensions=list(opt_params.values()),
random_state=1,
n_initial_points=self.n_initial_points,
)
x = [optimizer.ask() for i in range(self.n_initial_points)]
ask = [dict(zip(opt_params.keys(), val)) for val in x]
with self.output()["opt"].localize("w") as tmp:
tmp.dump(optimizer)
with self.output()["todos"].localize("w") as tmp:
tmp.dump(ask, cls=NumpyEncoder)
with self.output()["keys"].localize("w") as tmp:
tmp.dump(list(opt_params.keys()))
def __init__(self, comm, num_blocks, opt_params):
print("Coordinator initializing")
self.comm = comm
self.num_blocks = num_blocks
self.opt_params = opt_params
self.optimizer = skopt.Optimizer(dimensions=self.opt_params,
random_state=13579)
self.param_list = []
self.fom_list = []
self.block_dict = {}
self.req_dict = {}
self.best_params = None
self.best_fom = None
self.next_params = []
self.to_tell = []
self.ends_cycle = False
self.target_fom = None
def _setup_skopt(self):
if self._points_to_evaluate and isinstance(self._points_to_evaluate, list):
if isinstance(self._points_to_evaluate[0], list):
# Keep backwards compatibility
self._points_to_evaluate = [
dict(zip(self._parameter_names, point))
for point in self._points_to_evaluate
]
# Else: self._points_to_evaluate is already in correct format
validate_warmstart(
self._parameter_names, self._points_to_evaluate, self._evaluated_rewards
)
if not self._skopt_opt:
if not self._space:
raise ValueError(
"If you don't pass an optimizer instance to SkOptSearch, "
"pass a valid `space` parameter."
)
self._skopt_opt = sko.Optimizer(self._parameter_ranges)
if self._points_to_evaluate and self._evaluated_rewards:
skopt_points = [
[point[par] for par in self._parameter_names]
for point in self._points_to_evaluate
]
self._skopt_opt.tell(skopt_points, self._evaluated_rewards)
elif self._points_to_evaluate:
self._initial_points = self._points_to_evaluate
self._parameters = self._parameter_names
# Skopt internally minimizes, so "max" => -1
if self._mode == "max":
self._metric_op = -1.0
elif self._mode == "min":
self._metric_op = 1.0
if self._metric is None and self._mode:
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
def optimal_allocation_with_skopt(t,
X,
Y,
n=10,
n_parallel=4,
const_income=True):
# [0, 1]
nn = 2
opt_fun = _fun
if const_income:
nn = 1
opt_fun = _fun_constant_income
dimensions = [Real(0, 1)] * nn * (t - 1)
optimizer = skopt.Optimizer(
dimensions,
base_estimator='gp',
random_state=1
# n_random_starts=None, n_initial_points=10, acq_func='gp_hedge', acq_optimizer='auto', acq_func_kwargs=None, acq_optimizer_kwargs=None
)
# fun = functools.partial(_fun, t, X, Y)
fun = opt_fun(t, X, Y)
if n_parallel <= 1:
print('not parallel')
for i in range(n):
suggested = optimizer.ask()
y = fun(suggested)
optimizer.tell(suggested, y)
print('iteration: {}, {}, {}'.format(i, suggested, y))
else:
# something not working here
print('parallel')
n_left = n
for i in range(0, n, max(n_parallel, 1)):
suggested = optimizer.ask(n_points=min(n_left, n_parallel))
n_left -= n_parallel
print(n_left)
y = Parallel()(delayed(fun)(x) for x in suggested)
optimizer.tell(suggested, y)
print('iteration: {}, {}, {}, {}'.format(
i, suggested, y, action_to_zeroone(np.array(suggested))))
print('min is', min(optimizer.yi))
return optimizer
def init_skopt(self, opt_initial_points):
"""Initialize as scikit-optimize (skopt) Optimizer with a 5-dimensional search space
Aligned with skopt ask-tell design pattern (https://geekyisawesome.blogspot.com/2018/07/hyperparameter-tuning-using-scikit.html)
Args:
opt_initial_points (int): number of random initial points for the optimizer
"""
self.opt = skopt.Optimizer(
[
skopt.space.Categorical(AUG_TYPES, name="A_aug1_type"),
skopt.space.Real(0.0, 1.0, name="A_aug1_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="A_aug2_type"),
skopt.space.Real(0.0, 1.0, name="A_aug2_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="B_aug1_type"),
skopt.space.Real(0.0, 1.0, name="B_aug1_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="B_aug2_type"),
skopt.space.Real(0.0, 1.0, name="B_aug2_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="C_aug1_type"),
skopt.space.Real(0.0, 1.0, name="C_aug1_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="C_aug2_type"),
skopt.space.Real(0.0, 1.0, name="C_aug2_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="D_aug1_type"),
skopt.space.Real(0.0, 1.0, name="D_aug1_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="D_aug2_type"),
skopt.space.Real(0.0, 1.0, name="D_aug2_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="E_aug1_type"),
skopt.space.Real(0.0, 1.0, name="E_aug1_magnitude"),
skopt.space.Categorical(AUG_TYPES, name="E_aug2_type"),
skopt.space.Real(0.0, 1.0, name="E_aug2_magnitude")
],
n_initial_points=opt_initial_points,
base_estimator="RF", # Random Forest estimator
acq_func="EI", # Expected Improvement
acq_optimizer="auto",
random_state=0,
)
def __init__(self, search_space: Dict[str, distributions.BaseDistribution],
skopt_kwargs: Dict[str, Any]) -> None:
self._search_space = search_space
dimensions = []
for name, distribution in sorted(self._search_space.items()):
if isinstance(distribution, distributions.IntDistribution):
if distribution.log:
low = distribution.low - 0.5
high = distribution.high + 0.5
dimension = space.Real(low, high, prior="log-uniform")
else:
count = (distribution.high -
distribution.low) // distribution.step
dimension = space.Integer(0, count)
elif isinstance(distribution,
distributions.CategoricalDistribution):
dimension = space.Categorical(distribution.choices)
elif isinstance(distribution, distributions.FloatDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
if distribution.log:
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low,
high,
prior="log-uniform")
elif distribution.step is not None:
count = int((distribution.high - distribution.low) //
distribution.step)
dimension = space.Integer(0, count)
else:
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high)
else:
raise NotImplementedError(
"The distribution {} is not implemented.".format(
distribution))
dimensions.append(dimension)
self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def get_optimizer(rand_init_num=20):
aug_name_list = get_augment_name_list()
# define the skopt optimizer
opt = skopt.Optimizer(
[
skopt.space.Categorical(aug_name_list, name="A_aug_type"),
skopt.space.Real(0.0, 1.0, name="A_aug_probability"),
skopt.space.Real(0.0, 1.0, name="A_aug_magnitude"),
skopt.space.Categorical(aug_name_list, name="B_aug_type"),
skopt.space.Real(0.0, 1.0, name="B_aug_probability"),
skopt.space.Real(0.0, 1.0, name="B_aug_magnitude"),
],
n_initial_points=rand_init_num,
base_estimator='RF',
acq_func='EI',
acq_optimizer='auto',
random_state=int(time.time()),
)
return opt
def _skopt_engine(self, error_function, initial_guess, maxiter,
optimizer_name):
if not SKOPT_AVAILABLE:
logger.info('Falling back on scipy opt engine')
self._scipy_engine(error_function, initial_guess, maxiter,
optimizer_name)
# define parameter hyperbox
parameter_box = self.define_parameter_hyperbox(initial_guess,
self._lower_boxsize,
self._upper_boxsize,
self._zero_boxsize)
# run optimization procedure (sequential model-based optimization)
optimizer = skopt.Optimizer(dimensions=parameter_box,
base_estimator='GP',
n_initial_points=10)
parameter_error_pairs = self._gridsearch_engine(
error_function, initial_guess)
for parameter_guess, error_value in parameter_error_pairs:
optimizer.tell(list(parameter_guess), error_value)
for _ in range(maxiter):
suggested = optimizer.ask()
error_value = error_function(suggested)
optimizer.tell(suggested, error_value)
def getSuggestions(self, search_space, trials, request_number):
"""
Get the new suggested trials with skopt algorithm.
"""
skopt_search_space = []
for param in search_space.params:
if param.type == INTEGER:
skopt_search_space.append(skopt.space.Integer(
int(param.min), int(param.max), name=param.name))
elif param.type == DOUBLE:
skopt_search_space.append(skopt.space.Real(
float(param.min), float(param.max), "log-uniform", name=param.name))
elif param.type == CATEGORICAL or param.type == DISCRETE:
skopt_search_space.append(
skopt.space.Categorical(param.list, name=param.name))
if self.algorithm_name != "bayesianoptimization":
raise Exception(
'"Failed to create the algortihm: {}'.format(self.algorithm_name))
skopt_optimizer = skopt.Optimizer(skopt_search_space,
base_estimator=self.base_estimator,
n_initial_points=self.n_initial_points,
acq_func=self.acq_func,
acq_optimizer=self.acq_optimizer,
random_state=self.random_state)
skopt_suggested = []
loss_for_skopt = []
for trial in trials:
trial_assignment = []
for param in search_space.params:
parameter_value = None
for assignment in trial.assignments:
if assignment.name == param.name:
parameter_value = assignment.value
break
if param.type == INTEGER:
trial_assignment.append(int(parameter_value))
elif param.type == DOUBLE:
trial_assignment.append(float(parameter_value))
else:
trial_assignment.append(parameter_value)
skopt_suggested.append(trial_assignment)
loss_value = float(trial.target_metric.value)
if search_space.goal == MAX_GOAL:
loss_value = -1 * loss_value
loss_for_skopt.append(loss_value)
if loss_for_skopt != [] and skopt_suggested != []:
skopt_optimizer.tell(skopt_suggested, loss_for_skopt)
return_trial_list = []
for i in range(request_number):
skopt_suggested = skopt_optimizer.ask()
return_trial_list.append(
BaseSkoptService.convert(search_space, skopt_suggested))
return return_trial_list
async def main():
args = parser.parse_args()
if args.debug:
if args.debug == sys.stdout:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.DEBUG,
filename=args.debug,
filemode='w')
# Do not run the tuner if something is wrong with the adjudication option
# that is set by the user. These options could be critical in tuning.
win_adj_count, win_adj_score = 4, Arena.MATE_SCORE
if args.win_adj:
for n in args.win_adj:
m = re.match('count=(\d+)', n)
if m:
win_adj_count = int(m.group(1))
m = re.match('score=(\d+)', n)
if m:
win_adj_score = int(m.group(1))
book = []
if args.book:
book.extend(load_book(args.book, args.n_book))
print(f'Loaded book with {len(book)} positions')
if not book:
book.append(chess.Board())
print('No book. Starting every game from initial position.')
print('Loading engines')
conf = load_conf(args.conf)
engines = await asyncio.gather(
*(asyncio.gather(load_engine(conf, args.engine),
load_engine(conf, args.opp_engine or args.engine))
for _ in range(args.concurrency)))
options = engines[0][0].options
print('Parsing options')
dim_names, dimensions = parse_options(args.opt, args.c_opt, options)
opt = skopt.Optimizer(dimensions,
base_estimator=args.base_estimator,
n_initial_points=args.n_initial_points,
acq_func=args.acq_func,
acq_optimizer=args.acq_optimizer,
acq_func_kwargs={
'xi': args.acq_xi,
'kappa': args.acq_kappa,
'noise': args.acq_noise,
'n_jobs': -1,
},
acq_optimizer_kwargs=dict(
n_points=args.acq_n_points, ))
if args.games_file:
games_file = args.games_file.open('a')
else:
games_file = sys.stdout
if args.log_file:
key_args = {}
# Not all arguments change the result, so no need to keep them in the key.
for arg_group in (games_group, tune_group, engine_group):
for arg in arg_group._group_actions:
key = arg.dest
key_args[key] = getattr(args, key)
key = repr(sorted(key_args.items())).encode()
data_logger = DataLogger(args.log_file,
key=hashlib.sha256(key).hexdigest())
cached_games = data_logger.load(opt)
else:
print('No -log-file set. Results won\'t be saved.')
data_logger = None
cached_games = 0
limit = chess.engine.Limit(nodes=args.nodes,
time=args.movetime and args.movetime / 1000)
assert args.games_per_encounter >= 2 and args.games_per_encounter % 2 == 0, \
'Games per encounter must be even and >= 2.'
# Run tasks concurrently
try:
started = cached_games
def on_done(task):
if task.exception():
logging.error('Error while excecuting game')
task.print_stack()
def new_game(arena):
nonlocal started
x = opt.ask()
engine_args = x_to_args(x, dim_names, options)
print(
f'Starting {args.games_per_encounter} games {started}/{args.n} with {engine_args}'
)
async def routine():
await arena.configure(engine_args)
return await arena.run_games(
book,
game_id=started,
games_played=args.games_per_encounter)
task = asyncio.create_task(routine())
# We tag the task with some attributes that we need when it finishes.
setattr(task, 'tune_x', x)
setattr(task, 'tune_arena', arena)
setattr(task, 'tune_game_id', started)
task.add_done_callback(on_done)
started += 1
return task
tasks = []
if args.n - started > 0:
# TODO: Even though it knows we are asking for multiple points,
# we still often get all points equal. Specially if restarting
# from logged data. Maybe we should just sample random points ourselves.
xs = opt.ask(min(args.concurrency, args.n - started))
else:
xs = []
for conc_id, x_init in enumerate(xs):
enginea, engineb = engines[conc_id]
arena = Arena(enginea, engineb, limit, args.max_len, win_adj_count,
win_adj_score)
tasks.append(new_game(arena))
while tasks:
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
tasks = list(pending)
for task in done:
res = task.result()
arena, x, game_id = task.tune_arena, task.tune_x, task.tune_game_id
games, y, er = res
for game in games:
print(game, end='\n\n', file=games_file, flush=True)
if er:
print('Game erred:', er, type(er))
continue
opt.tell(x, -y) # opt is minimizing
# Delete old models to save memory. Note that for the first 10 tells (default)
# or the specified -n-initial-points no model is created, as we are still just
# querying at random.
logging.debug(f'Number of models {len(opt.models)}')
if len(opt.models) > 1:
del opt.models[0]
results = ', '.join(g.headers['Result'] for g in games)
print(f'Finished game {game_id} {x} => {y} ({results})')
if data_logger:
data_logger.store(x, y)
if started < args.n:
tasks.append(new_game(arena))
if opt.models and game_id != 0 and args.result_interval > 0 and game_id % args.result_interval == 0:
summarize(opt, steps=1)
except asyncio.CancelledError:
pass
if opt.Xi and opt.models:
summarize(opt, steps=6)
if len(dimensions) == 1:
plot_optimizer(opt, dimensions[0].low, dimensions[0].high)
else:
print('Not enought data to summarize results.')
logging.debug('Quitting engines')
try:
# Could also use wait here, but wait for some reason fails if the list
# is empty. Why can't we just wait for nothing?
await asyncio.gather(*(e.quit() for es in engines for e in es
if not e.returncode.done()))
except chess.engine.EngineError:
pass
def doGGC():
gene_graphs = [
data.gene_graphs.OntologyGraph(neighbors=30,
embeddings_name='dl2vec',
randomize=False,
gene_names=list(features.columns),
relabel_genes=False),
]
for graph in gene_graphs:
adj = graph.adj()
for dropout in [False]: #, False]:
import gc
gc.collect()
skopt_args = collections.OrderedDict()
skopt_args["lr"] = Integer(3, 5)
skopt_args["channels"] = Integer(3, 6)
# skopt_args["embedding"]=Integer(4, 5)
skopt_args["num_layer"] = Integer(1, 3)
skopt_args["gat_heads"] = Integer(1, 3)
skopt_args["prepool_extralayers"] = Integer(0, 1)
optimizer = skopt.Optimizer(dimensions=skopt_args.values(),
base_estimator="GP",
n_initial_points=4,
random_state=args.seed)
print(skopt_args)
best_valid_metric = 0
test_for_best_valid_metric = 0
best_config = None
already_done = set()
for i in range(100):
import gc
gc.collect()
suggestion = optimizer.ask()
if str(suggestion) in already_done:
continue
already_done.add(str(suggestion))
sdict = dict(zip(skopt_args.keys(), suggestion))
sdict["lr"] = 10**float((-sdict["lr"]))
sdict["channels"] = 2**sdict["channels"]
sdict["gat_heads"] = 2**sdict["gat_heads"]
sdict["embedding"] = 2 # 2**sdict["embedding"]
print(sdict)
neighbors = graph.nx_graph
intersection_nodes = np.intersect1d(X_train.columns,
neighbors.nodes)
x_train = X_train[list(intersection_nodes)].copy()
x_valid = X_valid[list(intersection_nodes)].copy()
toremove = set(neighbors.nodes)
toremove = toremove.difference(intersection_nodes)
neighbors.remove_nodes_from(toremove)
adj = sparse.csr_matrix(nx.to_numpy_matrix(neighbors))
model = models.gnn.GCN(
name="GAT",
dropout=dropout,
gnn="GAT",
gat_heads=sdict["gat_heads"],
cuda=torch.cuda.is_available(),
num_layer=sdict["num_layer"],
prepool_extralayers=sdict["prepool_extralayers"],
channels=sdict["channels"],
embedding=sdict["channels"], #sdict["embedding"],
aggregation=None,
lr=sdict["lr"],
num_epochs=100,
patience=40,
verbose=True,
seed=args.seed)
try:
model.fit(x_train, y_train, adj)
with torch.no_grad():
model.eval()
y_valid_pred = model.predict(x_valid)
valid_metric = sklearn.metrics.accuracy_score(
y_valid, np.argmax(y_valid_pred, axis=1))
opt_results = optimizer.tell(suggestion, -valid_metric)
# #record metrics to write and plot
# if best_valid_metric < valid_metric:
# best_valid_metric = valid_metric
# print("best_valid_metric", best_valid_metric, sdict)
# best_config = sdict
# y_test_pred = model.predict(x_test)
# test_metric = sklearn.metrics.accuracy_score(y_test, np.argmax(y_test_pred,axis=1))
# test_for_best_valid_metric = test_metric
experiment = {
"model": model.name,
"graph": graph.graph_name,
"num_genes": len(x_train.columns),
"train_size": args.ntrain,
"seed": args.seed,
"acc": valid_metric,
'lr': sdict["lr"],
'channels': sdict["channels"],
'embedding': sdict["embedding"],
'num_layer': sdict["num_layer"],
'prepool_extralayers': sdict["prepool_extralayers"]
}
print(i, "This result:", valid_metric, experiment)
global results
results = record_result(results, experiment, filename)
except Exception as e:
print(e)
logging.error(logging.traceback.format_exc())
# cleanup
model.best_model = None
del model
torch.cuda.empty_cache()
print("#Final Results", test_for_best_valid_metric, best_config)
return test_for_best_valid_metric, best_config
def doMLP():
skopt_args = collections.OrderedDict()
skopt_args["lr"] = Integer(2, 5)
skopt_args["channels"] = Integer(4, 12)
skopt_args["layers"] = Integer(1, 4)
optimizer = skopt.Optimizer(dimensions=skopt_args.values(),
base_estimator="GP",
n_initial_points=3,
random_state=args.seed)
print(skopt_args)
best_valid_metric = 0
test_for_best_valid_metric = 0
best_config = None
already_done = set()
for i in range(10):
suggestion = optimizer.ask()
if str(suggestion) in already_done:
continue
already_done.add(str(suggestion))
sdict = dict(zip(skopt_args.keys(), suggestion))
sdict["lr"] = 10**float((-sdict["lr"]))
sdict["channels"] = 2**sdict["channels"]
model = models.mlp.MLP(name="MLP",
num_layer=sdict["layers"],
channels=sdict["channels"],
lr=sdict["lr"],
num_epochs=100,
patience=50,
cuda=torch.cuda.is_available(),
metric=sklearn.metrics.accuracy_score,
verbose=False,
seed=args.seed)
model.fit(X_train, y_train)
y_valid_pred = model.predict(X_valid)
valid_metric = sklearn.metrics.accuracy_score(
y_valid, np.argmax(y_valid_pred, axis=1))
opt_results = optimizer.tell(suggestion, -valid_metric)
print(opt_results)
#record metrics to write and plot
if best_valid_metric < valid_metric:
best_valid_metric = valid_metric
best_config = sdict
y_test_pred = model.predict(X_test)
test_metric = sklearn.metrics.accuracy_score(
y_test, np.argmax(y_test_pred, axis=1))
test_for_best_valid_metric = test_metric
print(i, "This result:", valid_metric, sdict)
experiment = {
"model": model.name,
"graph": "",
"num_genes": len(list(X_train.columns)),
"train_size": args.ntrain,
"seed": args.seed,
"acc": valid_metric,
'lr': sdict["lr"],
'channels': sdict["channels"],
'embedding': 0,
'num_layer': sdict["layers"],
'prepool_extralayers': 0
}
global results
results = record_result(results, experiment, filename)
print("#Final Results", test_for_best_valid_metric, best_config)
return test_metric, best_config
def get_new_suggestions(self, study_name, input_trials=[], number=1):
"""
Get the new suggested trials with skopt algorithm.
"""
# Construct search space, example: {"x": hyperopt.hp.uniform('x', -10, 10), "x2": hyperopt.hp.uniform('x2', -10, 10)}
hyperopt_search_space = {}
study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
skopt_search_space = []
for param in params:
param_name = param["parameterName"]
if param["type"] == "INTEGER":
skopt_search_space.append(
skopt.space.Integer(
param["minValue"], param["maxValue"], name="min_samples_leaf"
)
)
elif param["type"] == "DOUBLE":
skopt_search_space.append(
skopt.space.Real(
param["minValue"],
param["maxValue"],
"log-uniform",
name="learning_rate",
)
)
elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
pass
if self.algorithm_name == "bayesian_optimization":
skopt_optimizer = skopt.Optimizer([(-2.0, 2.0)])
else:
print("Unsupport skopt algorithm: {}".format(self.algorithm_name))
completed_advisor_trials = Trial.objects.filter(
study_name=study_name, status="Completed"
)
for index, advisor_trial in enumerate(completed_advisor_trials):
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(advisor_trial.parameter_values)
# Example: [(-2.0, 2.0)]
skopt_suggested = []
for param in params:
if param["type"] == "INTEGER" or param["type"] == "DOUBLE":
parameter_value = parameter_values_json[param["parameterName"]]
skopt_suggested.append(parameter_value)
elif param["type"] == "DISCRETE":
pass
elif param["type"] == "CATEGORICAL":
pass
loss_for_skopt = advisor_trial.objective_value
if study_configuration_json["goal"] == "MAXIMIZE":
# Now hyperopt only supports fmin and we need to reverse objective value for maximization
loss_for_skopt = -1 * advisor_trial.objective_value
skopt_optimizer.tell(skopt_suggested, loss_for_skopt)
return_trial_list = []
for i in range(number):
skopt_suggested = skopt_optimizer.ask()
new_advisor_trial = Trial.create(study.name, "SkoptTrial")
parameter_values_json = {}
index = 0
for param in params:
if param["type"] == "INTEGER" or param["type"] == "DOUBLE":
parameter_values_json[param["parameterName"]] = skopt_suggested[
index
]
elif param["type"] == "DISCRETE":
pass
elif param["type"] == "CATEGORICAL":
pass
index += 1
new_advisor_trial.parameter_values = json.dumps(parameter_values_json)
new_advisor_trial.save()
return_trial_list.append(new_advisor_trial)
return return_trial_list
if not os.path.exists(result_name):
starfinder_obj = make_starfinder_objective(recipe, name)
starfinder_dims = [
Real(-6., 3, name='threshold'),
Real(2., 8, name='fwhm'),
Real(2., 5.5, name='sigma_radius'),
Real(-15., 0., name='roundlo'),
Real(0., 15., name='roundhi'),
Real(-15, 0., name='sharplo'),
Real(0., 10., name='sharphi')
]
starfinder_optimizer = skopt.Optimizer(
dimensions=starfinder_dims,
n_jobs=12,
random_state=1,
base_estimator='RF',
n_initial_points=1000,
initial_point_generator='random'
)
starfinder_result = run_optimizer(starfinder_optimizer, starfinder_obj, n_evaluations=1400, n_processes=12)
with open(name+'_starfinder_opt.pkl', 'wb') as f:
dill.dump(starfinder_result, f)
with open(name+'starfinder_params.txt', 'w') as f:
f.write(str(list(zip(starfinder_result.space.dimension_names, starfinder_result.x))))
else:
with open(result_name, 'rb') as f:
starfinder_result = dill.load(f)
x = starfinder_result.x
def doGGC():
if args.graph == "stringdb":
graph = data.gene_graphs.StringDBGraph(datastore="./data")
elif args.graph == "genemania":
graph = data.gene_graphs.GeneManiaGraph()
else:
print("unknown graph")
sys.exit(1)
adj = graph.adj()
import gc
gc.collect()
skopt_args = collections.OrderedDict()
skopt_args["lr"] = Integer(3, 5)
#skopt_args["channels"]=Integer(4, 5)
#skopt_args["embedding"]=Integer(4, 5)
skopt_args["num_layer"] = Integer(2, 5)
skopt_args["prepool_extralayers"] = Integer(1, 2)
optimizer = skopt.Optimizer(dimensions=skopt_args.values(),
base_estimator="GP",
n_initial_points=4,
random_state=args.seed)
print(skopt_args)
best_valid_metric = 0
test_for_best_valid_metric = 0
best_config = None
already_done = set()
for i in range(10):
import gc
gc.collect()
suggestion = optimizer.ask()
if str(suggestion) in already_done:
continue
already_done.add(str(suggestion))
sdict = dict(zip(skopt_args.keys(), suggestion))
sdict["lr"] = 10**float((-sdict["lr"]))
sdict["channels"] = 32 #2**sdict["channels"]
sdict["embedding"] = 32 #2**sdict["embedding"]
print(sdict)
model = models.gcn.GCN(
name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy",
dropout=False,
cuda=torch.cuda.is_available(),
num_layer=sdict["num_layer"],
prepool_extralayers=sdict["prepool_extralayers"],
channels=sdict["channels"],
embedding=sdict["embedding"],
aggregation="hierarchy",
lr=sdict["lr"],
num_epochs=100,
patience=20,
verbose=True,
seed=args.seed)
model.fit(X_train, y_train, adj)
y_valid_pred = model.predict(X_valid)
valid_metric = sklearn.metrics.accuracy_score(
y_valid, np.argmax(y_valid_pred, axis=1))
opt_results = optimizer.tell(suggestion, -valid_metric)
#record metrics to write and plot
if best_valid_metric < valid_metric:
best_valid_metric = valid_metric
print("best_valid_metric", best_valid_metric, sdict)
best_config = sdict
y_test_pred = model.predict(X_test)
test_metric = sklearn.metrics.accuracy_score(
y_test, np.argmax(y_test_pred, axis=1))
test_for_best_valid_metric = test_metric
print(i, "This result:", valid_metric, sdict)
print("#Final Results", test_for_best_valid_metric, best_config)
return test_for_best_valid_metric, best_config