def start(self, max_evals): """Starts a new pySOT run.""" self.history = [] self.proposals = [] # Symmetric Latin hypercube design des_pts = max([self.batch_size, 2 * (self.opt.dim + 1)]) slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts) # Warped RBF interpolant rbf = RBFInterpolant( dim=self.opt.dim, lb=self.opt.lb, ub=self.opt.ub, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4, ) # Optimization strategy self.strategy = SRBFStrategy( max_evals=self.max_evals, opt_prob=self.opt, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=1, use_restarts=True, )
def init(): print("\nInitializing run...") rbf = RBFInterpolant( dim=ackley.dim, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube( dim=ackley.dim, num_pts=2*(ackley.dim+1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy( max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) # Wrap controller in checkpoint object controller = CheckpointController(controller, fname=fname) result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def test_srbf_async(): max_evals = 200 rbf = RBFInterpolant(dim=ackley.dim, lb=ackley.lb, ub=ackley.ub, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=None) for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) controller.run() check_strategy(controller)
def init(): print("\nInitializing run...") rbf = RBFInterpolant(dim=ackley.dim, lb=ackley.lb, ub=ackley.ub, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Create a strategy and a controller controller = SerialController(ackley.eval) controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True) print("Number of workers: 1") print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Wrap controller in checkpoint object controller = CheckpointController(controller, fname=fname) result = controller.run() print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_extra_vals(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_extra_vals.log"): os.remove("./logfiles/example_extra_vals.log") logging.basicConfig(filename="./logfiles/example_extra_vals.log", level=logging.INFO) num_threads = 4 max_evals = 500 ackley = Ackley(dim=10) num_extra = 10 extra = np.random.uniform(ackley.lb, ackley.ub, (num_extra, ackley.dim)) extra_vals = np.nan * np.ones((num_extra, 1)) for i in range(num_extra): # Evaluate every second point if i % 2 == 0: extra_vals[i] = ackley.eval(extra[i, :]) rbf = RBFInterpolant(dim=ackley.dim, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2*(ackley.dim+1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy( max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_threads, extra_points=extra, extra_vals=extra_vals) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Append the known function values to the POAP database since # POAP won't evaluate these points for i in range(len(extra_vals)): if not np.isnan(extra_vals[i]): record = EvalRecord( params=(np.ravel(extra[i, :]),), status='completed') record.value = extra_vals[i] record.feasible = True controller.fevals.append(record) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_matlab_engine(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_matlab_engine.log"): os.remove("./logfiles/example_matlab_engine.log") logging.basicConfig(filename="./logfiles/example_matlab_engine.log", level=logging.INFO) num_threads = 4 max_evals = 500 ackley = Ackley(dim=10) rbf = RBFInterpolant(dim=ackley.dim, lb=ackley.lb, ub=ackley.ub, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Use the serial controller (uses only one thread) controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Launch the threads for _ in range(num_threads): try: worker = MatlabWorker(controller) worker.matlab = matlab.engine.start_matlab() controller.launch_worker(worker) except Exception as e: print("\nERROR: Failed to initialize a MATLAB session.\n") print(str(e)) return # Run the optimization strategy result = controller.run() # Print the final result print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_subprocess_files(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_subprocess_files.log"): os.remove("./logfiles/example_subprocess_files.log") logging.basicConfig(filename="./logfiles/example_subprocess_files.log", level=logging.INFO) print("\nNumber of threads: 4") print("Maximum number of evaluations: 200") print("Sampling method: Candidate DYCORS") print("Experimental design: Symmetric Latin Hypercube") print("Surrogate: Cubic RBF") assert os.path.isfile(path), "You need to build sphere_ext_files" num_threads = 4 max_evals = 200 sphere = Sphere(dim=10) rbf = RBFInterpolant(dim=sphere.dim, kernel=TPSKernel(), tail=LinearTail(sphere.dim)) slhd = SymmetricLatinHypercube(dim=sphere.dim, num_pts=2 * (sphere.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=sphere, exp_design=slhd, surrogate=rbf, asynchronous=False, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Launch the threads and give them access to the objective function for i in range(num_threads): worker = CppSim(controller) worker.my_filename = str(i) + ".txt" controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_simple(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_simple.log"): os.remove("./logfiles/example_simple.log") logging.basicConfig(filename="./logfiles/example_simple.log", level=logging.INFO) print("\nNumber of threads: 4") print("Maximum number of evaluations: 500") print("Sampling method: CandidateDYCORS") print("Experimental design: Symmetric Latin Hypercube") print("Surrogate: Cubic RBF") num_threads = 4 max_evals = 500 ackley = Ackley(dim=10) rbf = SurrogateUnitBox(RBFInterpolant(dim=ackley.dim, kernel=CubicKernel(), tail=LinearTail(ackley.dim)), lb=ackley.lb, ub=ackley.ub) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_subprocess_partial_info(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_subprocess_partial_info.log"): os.remove("./logfiles/example_subprocess_partial_info.log") logging.basicConfig( filename="./logfiles/example_subprocess_partial_info.log", level=logging.INFO) assert os.path.isfile(path), "You need to build sumfun_ext" num_threads = 4 max_evals = 200 sumfun = SumfunExt(dim=10) rbf = RBFInterpolant(dim=sumfun.dim, lb=sumfun.lb, ub=sumfun.ub, kernel=CubicKernel(), tail=LinearTail(sumfun.dim)) slhd = SymmetricLatinHypercube(dim=sumfun.dim, num_pts=2 * (sumfun.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=sumfun, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Launch the threads and give them access to the objective function for _ in range(num_threads): controller.launch_worker(CppSim(controller)) # Run the optimization strategy result = controller.run() print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def example_mars(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_mars.log"): os.remove("./logfiles/example_mars.log") logging.basicConfig(filename="./logfiles/example_mars.log", level=logging.INFO) num_threads = 4 max_evals = 200 ackley = Ackley(dim=5) try: mars = MARSInterpolant(dim=ackley.dim, lb=ackley.lb, ub=ackley.ub) except Exception as e: print(str(e)) return slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=mars, asynchronous=True, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(mars.__class__.__name__)) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def main_master(num_workers): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/test_subprocess_mpi.log"): os.remove("./logfiles/test_subprocess_mpi.log") logging.basicConfig(filename="./logfiles/test_subprocess_mpi.log", level=logging.INFO) print("\nTesting the POAP MPI controller with {0} workers".format( num_workers)) print("Maximum number of evaluations: 200") print("Search strategy: Candidate DYCORS") print("Experimental design: Symmetric Latin Hypercube") print("Surrogate: Cubic RBF") assert os.path.isfile(path), "You need to build sphere_ext" max_evals = 200 sphere = Sphere(dim=10) rbf = RBFInterpolant(dim=sphere.dim, kernel=CubicKernel(), tail=LinearTail(sphere.dim)) slhd = SymmetricLatinHypercube(dim=sphere.dim, num_pts=2 * (sphere.dim + 1)) # Create a strategy and a controller strategy = SRBFStrategy(max_evals=max_evals, opt_prob=sphere, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_workers) controller = MPIController(strategy) print("Number of threads: {}".format(num_workers)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def test_srbf_serial(): max_evals = 200 rbf = RBFInterpolant( dim=ackley.dim, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube( dim=ackley.dim, num_pts=2*(ackley.dim+1)) # Create a strategy and a controller controller = SerialController(ackley.eval) controller.strategy = SRBFStrategy( max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True) controller.run() check_strategy(controller)
def test_example_simple(): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_simple.log"): os.remove("./logfiles/example_simple.log") logging.basicConfig(filename="./logfiles/example_simple.log", level=logging.INFO) num_threads = 2 max_evals = 50 ackley = Ackley(dim=10) rbf = RBFInterpolant(dim=ackley.dim, lb=ackley.lb, ub=ackley.ub, kernel=CubicKernel(), tail=LinearTail(ackley.dim)) slhd = SymmetricLatinHypercube(dim=ackley.dim, num_pts=2 * (ackley.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=ackley, exp_design=slhd, surrogate=rbf, asynchronous=True) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, ackley.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def main_master(opt_prob, num_workers): if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/mpiexample_mpi.log"): os.remove("./logfiles/mpiexample_mpi.log") logging.basicConfig(filename="./logfiles/mpiexample_mpi.log", level=logging.INFO) max_evals = 500 rbf = RBFInterpolant(dim=opt_prob.dim, lb=opt_prob.lb, ub=opt_prob.ub, kernel=CubicKernel(), tail=LinearTail(opt_prob.dim)) slhd = SymmetricLatinHypercube(dim=opt_prob.dim, num_pts=2 * (opt_prob.dim + 1)) # Create a strategy and a controller strategy = SRBFStrategy( max_evals=max_evals, opt_prob=opt_prob, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=num_workers, ) controller = MPIController(strategy) print("Number of workers: {}".format(num_workers)) print("Maximum number of evaluations: {}".format(max_evals)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(slhd.__class__.__name__)) print("Surrogate: {}".format(rbf.__class__.__name__)) result = controller.run() print("Best value found: {0}".format(result.value)) print("Best solution found: {0}\n".format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))
def pysot_cube(objective, scale, n_trials, n_dim, with_count=False): if False: if not os.path.exists("./logfiles"): os.makedirs("logfiles") if os.path.exists("./logfiles/example_simple.log"): os.remove("./logfiles/example_simple.log") logging.basicConfig(filename="./logfiles/example_simple.log", level=logging.INFO) num_threads = 2 max_evals = n_trials gp = GenericProblem(dim=n_dim, objective=objective, scale=scale) rbf = RBFInterpolant(dim=n_dim, lb=np.array([-scale] * n_dim), ub=np.array([scale] * n_dim), kernel=CubicKernel(), tail=LinearTail(n_dim)) slhd = SymmetricLatinHypercube(dim=n_dim, num_pts=2 * (n_dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=gp, exp_design=slhd, surrogate=rbf, asynchronous=True) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, gp.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() return (result.value, gp.feval_count) if with_count else result.value
class tuSOTOptimizer(AbstractOptimizer): primary_import = "pysot" def __init__(self, api_config): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.space_x = JointSpace(api_config) self.bounds = self.space_x.get_bounds() self.create_opt_prob() # Sets up the optimization problem (needs self.bounds) self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int self.turbo_batch_size = None self.pysot_batch_size = None self.history = [] self.proposals = [] self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1] self.dim = len(self.bounds) self.turbo = Turbo1( f=None, lb=self.bounds[:, 0], ub=self.bounds[:, 1], n_init=2 * self.dim + 1, max_evals=self.max_evals, batch_size=4, # We need to update this later verbose=False, ) self.meta_init = self.get_init_hp() def get_init_hp(self): api_config = self.api_config feature = get_api_config_feature(api_config) result, values = [], [] for i in range(9): model = joblib.load( './lgb_meta_model_{}.pkl'.format( i)) value = model.predict(np.array(feature).reshape(1, -1))[-1] values.append(value) result.append(math.e ** value) result_dict = {} for i, (k, v) in enumerate(api_config.items()): if v['type'] == "real": lower, upper = v["range"][0], v["range"][1] if lower < result[i] < upper: out = result[i] else: out = np.random.uniform(lower, upper) elif v['type'] == "int": lower, upper = v["range"][0], v["range"][1] if lower < result[i] < upper: out = int(result[i]) else: out = np.random.randint(lower, upper) elif v['type'] == "bool": out = True if int(result[i]) >= 1 else False else: out = api_config[k]["values"][0] result_dict[k] = out return result[:len(api_config)], values[:len(api_config)], result_dict def restart(self): self.turbo._restart() self.turbo._X = np.zeros((0, self.turbo.dim)) self.turbo._fX = np.zeros((0, 1)) X_init = latin_hypercube(self.turbo.n_init, self.dim) self.X_init = from_unit_cube(X_init, self.lb, self.ub) def create_opt_prob(self): """Create an optimization problem object.""" opt = OptimizationProblem() opt.lb = self.bounds[:, 0] # In warped space opt.ub = self.bounds[:, 1] # In warped space opt.dim = len(self.bounds) opt.cont_var = np.arange(len(self.bounds)) opt.int_var = [] assert len(opt.cont_var) + len(opt.int_var) == opt.dim opt.objfun = None self.opt = opt def start(self): """Starts a new pySOT run.""" self.history = [] self.proposals = [] # Symmetric Latin hypercube design des_pts = max([self.pysot_batch_size, 2 * (self.opt.dim + 1)]) slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts) # Warped RBF interpolant rbf = RBFInterpolant(dim=self.opt.dim, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4) rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub) # Optimization strategy self.strategy = SRBFStrategy( max_evals=self.max_evals, opt_prob=self.opt, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=1, use_restarts=True, ) def pysot_suggest(self, n_suggestions=1): if self.pysot_batch_size is None: # First call to suggest self.pysot_batch_size = n_suggestions self.start() # Set the tolerances pretending like we are running batch d, p = float(self.opt.dim), float(n_suggestions) self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p))) # Now we can make suggestions x_w = [] self.proposals = [] for _ in range(n_suggestions): proposal = self.strategy.propose_action() record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() # This triggers all the callbacks # It is possible that pySOT proposes a previously evaluated point # when all variables are integers, so we just abort in this case # since we have likely converged anyway. See PySOT issue #30. x = list(proposal.record.params) # From tuple to list x_unwarped, = self.space_x.unwarp(x) if x_unwarped in self.history: warnings.warn("pySOT proposed the same point twice") self.start() return self.suggest(n_suggestions=n_suggestions) # NOTE: Append unwarped to avoid rounding issues self.history.append(copy(x_unwarped)) self.proposals.append(proposal) x_w.append(copy(x_unwarped)) return x_w def turbo_suggest(self, n_suggestions=1): if self.turbo_batch_size is None: # Remember the batch size on the first call to suggest self.turbo_batch_size = n_suggestions self.turbo.batch_size = n_suggestions self.turbo.failtol = np.ceil(np.max([4.0 / self.turbo_batch_size, self.dim / self.turbo_batch_size])) self.turbo.n_init = max([self.turbo.n_init, self.turbo_batch_size]) self.restart() X_next = np.zeros((n_suggestions, self.dim)) # Pick from the initial points n_init = min(len(self.X_init), n_suggestions) if n_init > 0: X_next[:n_init] = deepcopy(self.X_init[:n_init, :]) self.X_init = self.X_init[n_init:, :] # Remove these pending points # Get remaining points from TuRBO n_adapt = n_suggestions - n_init if n_adapt > 0: if len(self.turbo._X) > 0: # Use random points if we can't fit a GP X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub) fX = copula_standardize(deepcopy(self.turbo._fX).ravel()) # Use Copula X_cand, y_cand, _ = self.turbo._create_candidates( X, fX, length=self.turbo.length, n_training_steps=100, hypers={} ) X_next[-n_adapt:, :] = self.turbo._select_candidates(X_cand, y_cand)[:n_adapt, :] X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :], self.lb, self.ub) # Unwarp the suggestions suggestions = self.space_x.unwarp(X_next) suggestions[-1] = self.meta_init[2] return suggestions def suggest(self, n_suggestions=1): if n_suggestions == 1: return self.turbo_suggest(n_suggestions) else: suggestion = n_suggestions // 2 return self.turbo_suggest(suggestion) + self.pysot_suggest(n_suggestions - suggestion) def _observe(self, x, y): # Find the matching proposal and execute its callbacks idx = [x == xx for xx in self.history] if np.any(idx): i = np.argwhere(idx)[0].item() # Pick the first index if there are ties proposal = self.proposals[i] proposal.record.complete(y) self.proposals.pop(i) self.history.pop(i) def observe(self, X, y): """Send an observation of a suggestion back to the optimizer. Parameters ---------- X : list of dict-like Places where the objective function has already been evaluated. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. y : array-like, shape (n,) Corresponding values where objective has been evaluated """ assert len(X) == len(y) for x_, y_ in zip(X, y): # Just ignore, any inf observations we got, unclear if right thing if np.isfinite(y_): self._observe(x_, y_) XX, yy = self.space_x.warp(X), np.array(y)[:, None] if len(self.turbo._fX) >= self.turbo.n_init: self.turbo._adjust_length(yy) self.turbo.n_evals += self.turbo_batch_size self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX))) self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy))) self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX))) self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy))) # Check for a restart if self.turbo.length < self.turbo.length_min: self.restart()
def setup_backend( self, params, strategy="SRBF", surrogate="RBF", design=None, ): self.opt_problem = BBoptOptimizationProblem(params) design_kwargs = dict(dim=self.opt_problem.dim) _coconut_case_match_to_1 = design _coconut_case_match_check_1 = False if _coconut_case_match_to_1 is None: _coconut_case_match_check_1 = True if _coconut_case_match_check_1: self.exp_design = EmptyExperimentalDesign(**design_kwargs) if not _coconut_case_match_check_1: if _coconut_case_match_to_1 == "latin_hypercube": _coconut_case_match_check_1 = True if _coconut_case_match_check_1: self.exp_design = LatinHypercube(num_pts=2 * (self.opt_problem.dim + 1), **design_kwargs) if not _coconut_case_match_check_1: if _coconut_case_match_to_1 == "symmetric_latin_hypercube": _coconut_case_match_check_1 = True if _coconut_case_match_check_1: self.exp_design = SymmetricLatinHypercube( num_pts=2 * (self.opt_problem.dim + 1), **design_kwargs) if not _coconut_case_match_check_1: if _coconut_case_match_to_1 == "two_factorial": _coconut_case_match_check_1 = True if _coconut_case_match_check_1: self.exp_design = TwoFactorial(**design_kwargs) if not _coconut_case_match_check_1: _coconut_match_set_name_design_cls = _coconut_sentinel _coconut_match_set_name_design_cls = _coconut_case_match_to_1 _coconut_case_match_check_1 = True if _coconut_case_match_check_1: if _coconut_match_set_name_design_cls is not _coconut_sentinel: design_cls = _coconut_case_match_to_1 if _coconut_case_match_check_1 and not (callable(design_cls)): _coconut_case_match_check_1 = False if _coconut_case_match_check_1: self.exp_design = design_cls(**design_kwargs) if not _coconut_case_match_check_1: raise TypeError( "unknown experimental design {_coconut_format_0!r}".format( _coconut_format_0=(design))) surrogate_kwargs = dict(dim=self.opt_problem.dim, lb=self.opt_problem.lb, ub=self.opt_problem.ub) _coconut_case_match_to_2 = surrogate _coconut_case_match_check_2 = False if _coconut_case_match_to_2 == "RBF": _coconut_case_match_check_2 = True if _coconut_case_match_check_2: self.surrogate = RBFInterpolant( kernel=LinearKernel() if design is None else CubicKernel(), tail=ConstantTail(self.opt_problem.dim) if design is None else LinearTail(self.opt_problem.dim), **surrogate_kwargs) if not _coconut_case_match_check_2: if _coconut_case_match_to_2 == "GP": _coconut_case_match_check_2 = True if _coconut_case_match_check_2: self.surrogate = GPRegressor(**surrogate_kwargs) if not _coconut_case_match_check_2: _coconut_match_set_name_surrogate_cls = _coconut_sentinel _coconut_match_set_name_surrogate_cls = _coconut_case_match_to_2 _coconut_case_match_check_2 = True if _coconut_case_match_check_2: if _coconut_match_set_name_surrogate_cls is not _coconut_sentinel: surrogate_cls = _coconut_case_match_to_2 if _coconut_case_match_check_2 and not (callable(surrogate_cls)): _coconut_case_match_check_2 = False if _coconut_case_match_check_2: self.surrogate = surrogate_cls(**surrogate_kwargs) if not _coconut_case_match_check_2: raise TypeError("unknown surrogate {_coconut_format_0!r}".format( _coconut_format_0=(surrogate))) strategy_kwargs = dict(max_evals=sys.maxsize, opt_prob=self.opt_problem, exp_design=self.exp_design, surrogate=self.surrogate, asynchronous=True, batch_size=1) _coconut_case_match_to_3 = strategy _coconut_case_match_check_3 = False if _coconut_case_match_to_3 == "SRBF": _coconut_case_match_check_3 = True if _coconut_case_match_check_3: self.strategy = SRBFStrategy(**strategy_kwargs) if not _coconut_case_match_check_3: if _coconut_case_match_to_3 == "EI": _coconut_case_match_check_3 = True if _coconut_case_match_check_3: self.strategy = EIStrategy(**strategy_kwargs) if not _coconut_case_match_check_3: if _coconut_case_match_to_3 == "DYCORS": _coconut_case_match_check_3 = True if _coconut_case_match_check_3: self.strategy = DYCORSStrategy(**strategy_kwargs) if not _coconut_case_match_check_3: if _coconut_case_match_to_3 == "LCB": _coconut_case_match_check_3 = True if _coconut_case_match_check_3: self.strategy = LCBStrategy(**strategy_kwargs) if not _coconut_case_match_check_3: _coconut_match_set_name_strategy_cls = _coconut_sentinel _coconut_match_set_name_strategy_cls = _coconut_case_match_to_3 _coconut_case_match_check_3 = True if _coconut_case_match_check_3: if _coconut_match_set_name_strategy_cls is not _coconut_sentinel: strategy_cls = _coconut_case_match_to_3 if _coconut_case_match_check_3 and not (callable(strategy_cls)): _coconut_case_match_check_3 = False if _coconut_case_match_check_3: self.strategy = strategy_cls(**strategy_kwargs) if not _coconut_case_match_check_3: raise TypeError("unknown strategy {_coconut_format_0!r}".format( _coconut_format_0=(strategy)))
def run(self): """ Run the optimization @return: Nothing """ self.problem = VoltageOptimizationProblem( self.circuit, self.options, self.max_iter, callback=self.progress_signal.emit) # # (1) Optimization problem # # print(data.info) # # # (2) Experimental design # # Use a symmetric Latin hypercube with 2d + 1 samples # exp_des = SymmetricLatinHypercube(dim=self.problem.dim, npts=2 * self.problem.dim + 1) # # # (3) Surrogate model # # Use a cubic RBF interpolant with a linear tail # surrogate = RBFInterpolant(kernel=CubicKernel, tail=LinearTail, maxp=self.max_eval) # # # (4) Adaptive sampling # # Use DYCORS with 100d candidate points # adapt_samp = CandidateDYCORS(data=self, numcand=100 * self.dim) # # # Use the serial controller (uses only one thread) # controller = SerialController(self.objfunction) # # # (5) Use the sychronous strategy without non-bound constraints # strategy = SyncStrategyNoConstraints(worker_id=0, # data=self, # maxeval=self.max_eval, # nsamples=1, # exp_design=exp_des, # response_surface=surrogate, # sampling_method=adapt_samp) # # controller.strategy = strategy # # # Run the optimization strategy # result = controller.run() # # # Print the final result # print('Best value found: {0}'.format(result.value)) # print('Best solution found: {0}'.format(np.array_str(result.params[0], max_line_width=np.inf, precision=5, # suppress_small=True))) num_threads = 4 surrogate_model = GPRegressor(dim=self.problem.dim) sampler = SymmetricLatinHypercube(dim=self.problem.dim, num_pts=2 * (self.problem.dim + 1)) # Create a strategy and a controller controller = ThreadController() controller.strategy = SRBFStrategy(max_evals=self.max_iter, opt_prob=self.problem, exp_design=sampler, surrogate=surrogate_model, asynchronous=True, batch_size=num_threads) print("Number of threads: {}".format(num_threads)) print("Maximum number of evaluations: {}".format(self.max_iter)) print("Strategy: {}".format(controller.strategy.__class__.__name__)) print("Experimental design: {}".format(sampler.__class__.__name__)) print("Surrogate: {}".format(surrogate_model.__class__.__name__)) # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, self.problem.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=4, suppress_small=True))) self.solution = result.params[0] # Extract function values from the controller self.optimization_values = np.array( [o.value for o in controller.fevals]) # send the finnish signal self.progress_signal.emit(0.0) self.progress_text.emit('Done!') self.done_signal.emit()
class tuSOTOptimizer(AbstractOptimizer): primary_import = "pysot" def __init__(self, api_config): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.space_x = JointSpace(api_config) self.bounds = self.space_x.get_bounds() self.create_opt_prob( ) # Sets up the optimization problem (needs self.bounds) self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int self.turbo_batch_size = None self.pysot_batch_size = None self.history = [] self.proposals = [] self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1] self.dim = len(self.bounds) self.turbo = Turbo1( f=None, lb=self.bounds[:, 0], ub=self.bounds[:, 1], n_init=2 * self.dim + 1, max_evals=self.max_evals, batch_size=4, # We need to update this later verbose=False, ) # hyperopt self.random = np_random space, self.round_to_values = tuSOTOptimizer.get_hyperopt_dimensions( api_config) self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None) self.trials = Trials() # Some book keeping like opentuner wrapper self.trial_id_lookup = {} # Store just for data validation self.param_set_chk = frozenset(api_config.keys()) def restart(self): self.turbo._restart() self.turbo._X = np.zeros((0, self.turbo.dim)) self.turbo._fX = np.zeros((0, 1)) X_init = latin_hypercube(self.turbo.n_init, self.dim) self.X_init = from_unit_cube(X_init, self.lb, self.ub) def create_opt_prob(self): """Create an optimization problem object.""" opt = OptimizationProblem() opt.lb = self.bounds[:, 0] # In warped space opt.ub = self.bounds[:, 1] # In warped space opt.dim = len(self.bounds) opt.cont_var = np.arange(len(self.bounds)) opt.int_var = [] assert len(opt.cont_var) + len(opt.int_var) == opt.dim opt.objfun = None self.opt = opt def start(self): """Starts a new pySOT run.""" self.history = [] self.proposals = [] # Symmetric Latin hypercube design des_pts = max([self.pysot_batch_size, 2 * (self.opt.dim + 1)]) slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts) # Warped RBF interpolant rbf = RBFInterpolant(dim=self.opt.dim, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4) rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub) # Optimization strategy self.strategy = SRBFStrategy( max_evals=self.max_evals, opt_prob=self.opt, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=1, use_restarts=True, ) @staticmethod def hashable_dict(d): hashable_object = frozenset(d.items()) return hashable_object @staticmethod def get_hyperopt_dimensions(api_config): param_list = sorted(api_config.keys()) space = {} round_to_values = {} for param_name in param_list: param_config = api_config[param_name] param_type = param_config["type"] param_space = param_config.get("space", None) param_range = param_config.get("range", None) param_values = param_config.get("values", None) # Some setup for case that whitelist of values is provided: values_only_type = param_type in ("cat", "ordinal") if (param_values is not None) and (not values_only_type): assert param_range is None param_values = np.unique(param_values) param_range = (param_values[0], param_values[-1]) round_to_values[param_name] = interp1d( param_values, param_values, kind="nearest", fill_value="extrapolate") if param_type == "int": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.qloguniform(param_name, np.log(low), np.log(high), 1) else: space[param_name] = hp.quniform(param_name, low, high, 1) elif param_type == "bool": assert param_range is None assert param_values is None space[param_name] = hp.choice(param_name, (False, True)) elif param_type in ("cat", "ordinal"): assert param_range is None space[param_name] = hp.choice(param_name, param_values) elif param_type == "real": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.loguniform(param_name, np.log(low), np.log(high)) else: space[param_name] = hp.uniform(param_name, low, high) else: assert False, "type %s not handled in API" % param_type return space, round_to_values def get_trial(self, trial_id): for trial in self.trials._dynamic_trials: if trial["tid"] == trial_id: assert isinstance(trial, dict) # Make sure right kind of dict assert "state" in trial and "result" in trial assert trial["state"] == JOB_STATE_NEW return trial assert False, "No matching trial ID" def cleanup_guess(self, x_guess): assert isinstance(x_guess, dict) # Also, check the keys are only the vars we are searching over: assert frozenset(x_guess.keys()) == self.param_set_chk # Do the rounding # Make a copy to be safe, and also unpack singletons # We may also need to consider clip_chk at some point like opentuner x_guess = {k: only(x_guess[k]) for k in x_guess} for param_name, round_f in self.round_to_values.items(): x_guess[param_name] = round_f(x_guess[param_name]) # Also ensure this is correct dtype so sklearn is happy x_guess = { k: DTYPE_MAP[self.api_config[k]["type"]](x_guess[k]) for k in x_guess } return x_guess def pysot_suggest(self, n_suggestions=1): if self.pysot_batch_size is None: # First call to suggest self.pysot_batch_size = n_suggestions self.start() # Set the tolerances pretending like we are running batch d, p = float(self.opt.dim), float(n_suggestions) self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p))) # Now we can make suggestions x_w = [] self.proposals = [] for _ in range(n_suggestions): proposal = self.strategy.propose_action() record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() # This triggers all the callbacks # It is possible that pySOT proposes a previously evaluated point # when all variables are integers, so we just abort in this case # since we have likely converged anyway. See PySOT issue #30. x = list(proposal.record.params) # From tuple to list x_unwarped, = self.space_x.unwarp(x) if x_unwarped in self.history: warnings.warn("pySOT proposed the same point twice") self.start() return self.suggest(n_suggestions=n_suggestions) # NOTE: Append unwarped to avoid rounding issues self.history.append(copy(x_unwarped)) self.proposals.append(proposal) x_w.append(copy(x_unwarped)) return x_w def pysot_get_suggest(self, suggests): turbo_suggest_warps = self.space_x.warp(suggests) for i, warps in enumerate(turbo_suggest_warps): proposal = self.strategy.make_proposal(warps) proposal.add_callback(self.strategy.on_initial_proposal) record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() self.history.append(copy(suggests[i])) self.proposals.append(proposal) def turbo_suggest(self, n_suggestions=1): if self.turbo_batch_size is None: # Remember the batch size on the first call to suggest self.turbo_batch_size = n_suggestions self.turbo.batch_size = n_suggestions self.turbo.failtol = np.ceil( np.max([ 4.0 / self.turbo_batch_size, self.dim / self.turbo_batch_size ])) self.turbo.n_init = max([self.turbo.n_init, self.turbo_batch_size]) self.restart() X_next = np.zeros((n_suggestions, self.dim)) # Pick from the initial points n_init = min(len(self.X_init), n_suggestions) if n_init > 0: X_next[:n_init] = deepcopy(self.X_init[:n_init, :]) self.X_init = self.X_init[ n_init:, :] # Remove these pending points # Get remaining points from TuRBO n_adapt = n_suggestions - n_init if n_adapt > 0: if len(self.turbo._X ) > 0: # Use random points if we can't fit a GP X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub) fX = copula_standardize(deepcopy( self.turbo._fX).ravel()) # Use Copula X_cand, y_cand, _ = self.turbo._create_candidates( X, fX, length=self.turbo.length, n_training_steps=100, hypers={}) X_next[-n_adapt:, :] = self.turbo._select_candidates( X_cand, y_cand)[:n_adapt, :] X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :], self.lb, self.ub) # Unwarp the suggestions suggestions = self.space_x.unwarp(X_next) return suggestions def _hyperopt_suggest(self): new_ids = self.trials.new_trial_ids(1) assert len(new_ids) == 1 self.trials.refresh() seed = random_seed(self.random) new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed) assert len(new_trials) == 1 self.trials.insert_trial_docs(new_trials) self.trials.refresh() new_trial, = new_trials # extract singleton return new_trial def _hyperopt_transform(self, x): new_id = self.trials.new_trial_ids(1)[0] domain = self.domain rng = np.random.RandomState(1) idxs, vals = pyll.rec_eval(domain.s_idxs_vals, memo={ domain.s_new_ids: [new_id], domain.s_rng: rng, }) rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] rval_results = domain.new_result() for (k, _) in vals.items(): vals[k][0] = x[k] miscs_update_idxs_vals(rval_miscs, idxs, vals) rval_docs = self.trials.new_trial_docs([new_id], [None], rval_results, rval_miscs) return rval_docs[0] def hyperopt_suggest(self, n_suggestions=1): assert n_suggestions >= 1, "invalid value for n_suggestions" # Get the new trials, it seems hyperopt either uses random search or # guesses one at a time anyway, so we might as welll call serially. new_trials = [self._hyperopt_suggest() for _ in range(n_suggestions)] X = [] for trial in new_trials: x_guess = self.cleanup_guess(trial["misc"]["vals"]) X.append(x_guess) # Build lookup to get original trial object x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict" self.trial_id_lookup[x_guess_] = trial["tid"] assert len(X) == n_suggestions return X def hyperopt_get_suggest(self, suggests): trials = [self._hyperopt_transform(x) for x in suggests] for trial in trials: x_guess = self.cleanup_guess(trial["misc"]["vals"]) x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict" self.trial_id_lookup[x_guess_] = trial["tid"] self.trials.insert_trial_docs(trials) self.trials.refresh() def suggest(self, n_suggestions=1): if n_suggestions == 1: return self.turbo_suggest(n_suggestions) else: t_suggestion = n_suggestions // 2 # p_suggestion = int((n_suggestions - t_suggestion) * 3/4) h_suggestion = n_suggestions - t_suggestion turbo_suggest = self.turbo_suggest(t_suggestion) # pysot_suggest = self.pysot_suggest(p_suggestion) hyperopt_suggest = self.hyperopt_suggest(h_suggestion) self.hyperopt_get_suggest(turbo_suggest) # self.pysot_get_suggest(turbo_suggest + hyperopt_suggest) return turbo_suggest + hyperopt_suggest def _observe(self, x, y): # Find the matching proposal and execute its callbacks idx = [x == xx for xx in self.history] if np.any(idx): i = np.argwhere( idx)[0].item() # Pick the first index if there are ties proposal = self.proposals[i] proposal.record.complete(y) self.proposals.pop(i) self.history.pop(i) def observe(self, X, y): """Send an observation of a suggestion back to the optimizer. Parameters ---------- X : list of dict-like Places where the objective function has already been evaluated. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. y : array-like, shape (n,) Corresponding values where objective has been evaluated """ assert len(X) == len(y) # # pysot observe # for x_, y_ in zip(X, y): # # Just ignore, any inf observations we got, unclear if right thing # if np.isfinite(y_): # self._observe(x_, y_) # turbo observe XX, yy = self.space_x.warp(X), np.array(y)[:, None] if len(self.turbo._fX) >= self.turbo.n_init: self.turbo._adjust_length(yy) self.turbo.n_evals += self.turbo_batch_size self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX))) self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy))) self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX))) self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy))) # Check for a restart if self.turbo.length < self.turbo.length_min: self.restart() # hyperopt observe for x_guess, y_ in zip(X, y): x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ in self.trial_id_lookup, "Appears to be guess that did not originate from suggest" assert x_guess_ in self.trial_id_lookup, "trial object not available in trial dict" trial_id = self.trial_id_lookup.pop(x_guess_) trial = self.get_trial(trial_id) assert self.cleanup_guess( trial["misc"]["vals"] ) == x_guess, "trial ID not consistent with x values stored" # Cast to float to ensure native type result = {"loss": float(y_), "status": STATUS_OK} trial["state"] = JOB_STATE_DONE trial["result"] = result self.trials.refresh()
def pysot_cube(objective, n_trials, n_dim, with_count=False, method=None, design=None): """ Minimize :param objective: :param n_trials: :param n_dim: :param with_count: :return: """ logging.getLogger('pySOT').setLevel(logging.ERROR) num_threads = 1 asynchronous = True max_evals = n_trials gp = GenericProblem(dim=n_dim, objective=objective) if design == 'latin': exp_design = LatinHypercube(dim=n_dim, num_pts=2 * (n_dim + 1)) elif design == 'symmetric': exp_design = SymmetricLatinHypercube(dim=n_dim, num_pts=2 * (n_dim + 1)) elif design == 'factorial': exp_design = TwoFactorial(dim=n_dim) else: raise ValueError('design should be latin, symmetric or factorial') # Create a strategy and a controller # SRBFStrategy, EIStrategy, DYCORSStrategy,RandomStrategy, LCBStrategy controller = ThreadController() if method.lower() == 'srbf': surrogate = RBFInterpolant(dim=n_dim, lb=np.array([0.0] * n_dim), ub=np.array([1.0] * n_dim), kernel=CubicKernel(), tail=LinearTail(n_dim)) controller.strategy = SRBFStrategy(max_evals=max_evals, opt_prob=gp, exp_design=exp_design, surrogate=surrogate, asynchronous=asynchronous) elif method.lower() == 'ei': surrogate = GPRegressor(dim=n_dim, lb=np.array([0.0] * n_dim), ub=np.array([1.0] * n_dim)) controller.strategy = EIStrategy(max_evals=max_evals, opt_prob=gp, exp_design=exp_design, surrogate=surrogate, asynchronous=asynchronous) elif method.lower() == 'dycors': surrogate = RBFInterpolant(dim=n_dim, lb=np.array([0.0] * n_dim), ub=np.array([1.0] * n_dim), kernel=CubicKernel(), tail=LinearTail(n_dim)) controller.strategy = DYCORSStrategy(max_evals=max_evals, opt_prob=gp, exp_design=exp_design, surrogate=surrogate, asynchronous=asynchronous) elif method.lower() == 'lcb': surrogate = GPRegressor(dim=n_dim, lb=np.array([0.0] * n_dim), ub=np.array([1.0] * n_dim)) controller.strategy = LCBStrategy(max_evals=max_evals, opt_prob=gp, exp_design=exp_design, surrogate=surrogate, asynchronous=asynchronous) elif method.lower() == 'random': controller.strategy = RandomStrategy(max_evals=max_evals, opt_prob=gp) else: raise ValueError("Didn't recognize method passed to pysot") # Launch the threads and give them access to the objective function for _ in range(num_threads): worker = BasicWorkerThread(controller, gp.eval) controller.launch_worker(worker) # Run the optimization strategy result = controller.run() best_x = result.params[0].tolist() return (result.value, best_x, gp.feval_count) if with_count else (result.value, best_x)
class PySOTOptimizer(AbstractOptimizer): primary_import = "pysot" def __init__(self, api_config): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.space_x = JointSpace(api_config) self.bounds = self.space_x.get_bounds() self.create_opt_prob() # Sets up the optimization problem (needs self.bounds) self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int self.batch_size = None self.history = [] self.proposals = [] def create_opt_prob(self): """Create an optimization problem object.""" opt = OptimizationProblem() opt.lb = self.bounds[:, 0] # In warped space opt.ub = self.bounds[:, 1] # In warped space opt.dim = len(self.bounds) opt.cont_var = np.arange(len(self.bounds)) opt.int_var = [] assert len(opt.cont_var) + len(opt.int_var) == opt.dim opt.objfun = None self.opt = opt def start(self, max_evals): """Starts a new pySOT run.""" self.history = [] self.proposals = [] # Symmetric Latin hypercube design des_pts = max([self.batch_size, 2 * (self.opt.dim + 1)]) slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts) # Warped RBF interpolant rbf = RBFInterpolant(dim=self.opt.dim, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4) rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub) # Optimization strategy self.strategy = SRBFStrategy( max_evals=self.max_evals, opt_prob=self.opt, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=1, use_restarts=True, ) def suggest(self, n_suggestions=1): """Get a suggestion from the optimizer. Parameters ---------- n_suggestions : int Desired number of parallel suggestions in the output Returns ------- next_guess : list of dict List of `n_suggestions` suggestions to evaluate the objective function. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. """ if self.batch_size is None: # First call to suggest self.batch_size = n_suggestions self.start(self.max_evals) # Set the tolerances pretending like we are running batch d, p = float(self.opt.dim), float(n_suggestions) self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p))) # Now we can make suggestions x_w = [] self.proposals = [] for _ in range(n_suggestions): proposal = self.strategy.propose_action() record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() # This triggers all the callbacks # It is possible that pySOT proposes a previously evaluated point # when all variables are integers, so we just abort in this case # since we have likely converged anyway. See PySOT issue #30. x = list(proposal.record.params) # From tuple to list x_unwarped, = self.space_x.unwarp(x) if x_unwarped in self.history: warnings.warn("pySOT proposed the same point twice") self.start(self.max_evals) return self.suggest(n_suggestions=n_suggestions) # NOTE: Append unwarped to avoid rounding issues self.history.append(copy(x_unwarped)) self.proposals.append(proposal) x_w.append(copy(x_unwarped)) return x_w def _observe(self, x, y): # Find the matching proposal and execute its callbacks idx = [x == xx for xx in self.history] i = np.argwhere(idx)[0].item() # Pick the first index if there are ties proposal = self.proposals[i] proposal.record.complete(y) self.proposals.pop(i) self.history.pop(i) def observe(self, X, y): """Send an observation of a suggestion back to the optimizer. Parameters ---------- X : list of dict-like Places where the objective function has already been evaluated. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. y : array-like, shape (n,) Corresponding values where objective has been evaluated """ assert len(X) == len(y) for x_, y_ in zip(X, y): # Just ignore, any inf observations we got, unclear if right thing if np.isfinite(y_): self._observe(x_, y_)
def fit(self, X, y=None, **kwargs): """Run training with cross validation. :param X: training data :param **: parameters to be passed to GridSearchCV """ # wrap for pySOT class Target(OptimizationProblem): def __init__(self, outer): self.outer = outer param_def = outer.param_def self.lb = np.array([param['lb'] for param in param_def]) self.ub = np.array([param['ub'] for param in param_def]) self.dim = len(param_def) self.int_var = np.array([ idx for idx, param in enumerate(param_def) if param['integer'] ]) self.cont_var = np.array([ idx for idx, param in enumerate(param_def) if idx not in self.int_var ]) def eval_(self, x): print('Eval {0} ...'.format(x)) param_def = self.outer.param_def outer = self.outer # prepare parameters grid for gridsearchcv param_grid = ({ param['name']: [int(x[idx]) if param['integer'] else x[idx]] for idx, param in enumerate(param_def) }) # create gridsearchcv to evaluate the cv gs = GridSearchCV(outer.estimator, param_grid, refit=False, **outer.kwargs) # never refit during iteration, refit at the end gs.fit(X, y=y, **kwargs) gs_score = gs.best_score_ # gridsearchcv score is better when greater if not outer.best_score_ or gs_score > outer.best_score_: outer.best_score_ = gs_score outer.best_params_ = gs.best_params_ # also record history outer.params_history_.append(x) outer.score_history_.append(gs_score) print('Eval {0} => {1}'.format(x, gs_score)) # pySOT score is the lower the better, so return the negated return -gs_score # pySOT routine # TODO: make this configurable target = Target(self) rbf = SurrogateUnitBox(RBFInterpolant(dim=target.dim, kernel=CubicKernel(), tail=LinearTail(target.dim)), lb=target.lb, ub=target.ub) slhd = SymmetricLatinHypercube(dim=target.dim, num_pts=2 * (target.dim + 1)) # Create a strategy and a controller controller = SerialController(objective=target.eval_) controller.strategy = SRBFStrategy(max_evals=self.n_iter, batch_size=1, opt_prob=target, exp_design=slhd, surrogate=rbf, asynchronous=False) print('Maximum number of evaluations: {0}'.format(self.n_iter)) print('Strategy: {0}'.format(controller.strategy.__class__.__name__)) print('Experimental design: {0}'.format(slhd.__class__.__name__)) print('Surrogate: {0}'.format(rbf.__class__.__name__)) # Run the optimization strategy result = controller.run() print('Best value found: {0}'.format(result.value)) print('Best solution found: {0}\n'.format( np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)))