def _maximize(self,
runhistory: RunHistory,
stats: Stats,
num_points: int,
_sorted: bool = False,
**kwargs) -> List[Tuple[float, Configuration]]:
"""DifferentialEvolutionSolver
Parameters
----------
runhistory: ~dsmac.runhistory.runhistory.RunHistory
runhistory object
stats: ~dsmac.stats.stats.Stats
current stats object
num_points: int
number of points to be sampled
_sorted: bool
whether random configurations are sorted according to acquisition function
**kwargs
not used
Returns
-------
iterable
An iterable consistng of
tuple(acqusition_value, :class:`dsmac.configspace.Configuration`).
"""
from scipy.optimize._differentialevolution import DifferentialEvolutionSolver
configs = []
def func(x):
return -self.acquisition_function(
[Configuration(self.config_space, vector=x)])
ds = DifferentialEvolutionSolver(func,
bounds=[[0, 1], [0, 1]],
args=(),
strategy='best1bin',
maxiter=1000,
popsize=50,
tol=0.01,
mutation=(0.5, 1),
recombination=0.7,
seed=self.rng.randint(1000),
polish=True,
callback=None,
disp=False,
init='latinhypercube',
atol=0)
rval = ds.solve()
for pop, val in zip(ds.population, ds.population_energies):
rc = Configuration(self.config_space, vector=pop)
rc.origin = 'DifferentialEvolution'
configs.append((-val, rc))
configs.sort(key=lambda t: t[0])
configs.reverse()
return configs
def _call_ta(self,
config:Configuration,
instance:str,
instance_specific:str,
cutoff:float,
seed:int):
# TODO: maybe replace fixed instance specific and cutoff_length (0) to
# other value
cmd = []
cmd.extend(self.ta)
cmd.extend([instance, instance_specific, str(cutoff), "0", str(seed)])
for p in config:
if not config.get(p) is None:
cmd.extend(["-" + str(p), str(config[p])])
self.logger.debug("Calling: %s" % (" ".join(cmd)))
p = Popen(cmd, shell=False, stdout=PIPE, stderr=PIPE,
universal_newlines=True)
stdout_, stderr_ = p.communicate()
self.logger.debug("Stdout: %s" % (stdout_))
self.logger.debug("Stderr: %s" % (stderr_))
return stdout_, stderr_
def _call_ta(self, config: Configuration, instance: str,
instance_specific: str, cutoff: float, seed: int):
# TODO: maybe replace fixed instance specific and cutoff_length (0) to
# other value
cmd = []
cmd.extend(self.ta)
cmd.extend([
"--instance", instance, "--cutoff",
str(cutoff), "--seed",
str(seed), "--config"
])
for p in config:
if not config.get(p) is None:
cmd.extend(["-" + str(p), str(config[p])])
self.logger.debug("Calling: %s" % (" ".join(cmd)))
p = Popen(cmd,
shell=False,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
stdout_, stderr_ = p.communicate()
self.logger.debug("Stdout: %s" % (stdout_))
self.logger.debug("Stderr: %s" % (stderr_))
results = {"status": "CRASHED", "cost": 1234567890}
for line in stdout_.split("\n"):
if line.startswith("Result of this algorithm run:"):
fields = ":".join(line.split(":")[1:])
results = json.loads(fields)
return results, stdout_, stderr_
def fetch_new_runhistory(self, is_init=False):
if is_init:
query = self.Model.select().where(self.Model.origin >= 0)
else:
query = self.Model.select().where(self.Model.pid != os.getpid()).where(self.Model.origin >= 0)
config_cost=[]
for model in query:
config_id = model.config_id
config = model.config
config_bit = model.config_bit
config_origin = model.config_origin
cost = model.cost
time = model.time
instance_id = model.instance_id
seed = model.seed
status = model.status
additional_info = model.additional_info
origin = model.origin
timestamp = model.timestamp
try:
config = pickle.loads(config_bit)
except:
config = Configuration(self.config_space, values=json.loads(config), origin=config_origin)
try:
additional_info = json.loads(additional_info)
except Exception:
pass
if not self.runhistory.ids_config.get(config_id):
config_cost.append([config,cost])
self.runhistory.add(config, cost, time, StatusType(status), instance_id, seed, additional_info,
DataOrigin(origin))
self.timestamp = datetime.datetime.now()
return config_cost
def insert_runhistory(self,
config: Configuration,
cost: float,
time: float,
status: StatusType,
instance_id: str = "",
seed: int = 0,
additional_info: dict = {},
origin: DataOrigin = DataOrigin.INTERNAL):
config_id = get_id_of_config(config)
if instance_id is None:
instance_id = ""
try:
self.Model.create(config_id=config_id,
config=json.dumps(config.get_dictionary()),
config_origin=config.origin,
config_bit=pickle.dumps(config),
cost=cost,
time=time,
instance_id=instance_id,
seed=seed,
status=status.value,
additional_info=json.dumps(additional_info),
origin=origin.value,
pid=os.getpid(),
timestamp=datetime.datetime.now())
except pw.IntegrityError:
self.Model(
config_id=config_id,
config=json.dumps(config.get_dictionary()),
config_origin=config.origin,
config_bit=pickle.dumps(config),
cost=cost,
time=time,
instance_id=instance_id,
seed=seed,
status=status.value,
additional_info=json.dumps(additional_info),
origin=origin.value,
pid=os.getpid(),
timestamp=datetime.datetime.now(),
).save()
self.timestamp = datetime.datetime.now()
def load_json(self, fn: str, cs: ConfigurationSpace):
"""Load and runhistory in json representation from disk.
Overwrites current runhistory!
Parameters
----------
fn : str
file name to load from
cs : ConfigSpace
instance of configuration space
"""
try:
txt = self.file_system.read_txt(fn)
all_data = json.loads(txt, object_hook=StatusType.enum_hook)
except Exception as e:
self.logger.warning(
'Encountered exception %s while reading runhistory from %s. '
'Not adding any runs!',
e,
fn,
)
return
config_origins = all_data.get("config_origins", {})
self.ids_config = {}
self.ids_config = {(id_):
Configuration(cs,
values=values,
origin=config_origins.get(id_, None))
for id_, values in all_data["configs"].items()}
self.config_ids = {
config: id_
for id_, config in self.ids_config.items()
}
self._n_id = len(self.config_ids)
# important to use add method to use all data structure correctly
for k, v in all_data["data"]:
id_ = (k[0])
if id_ in self.ids_config:
self.add(config=self.ids_config[id_],
cost=float(v[0]),
time=float(v[1]),
status=StatusType(v[2]),
instance_id=k[1],
seed=int(k[2]),
additional_info=v[3])
def _compare_configs(self,
incumbent: Configuration,
challenger: Configuration,
run_history: RunHistory,
aggregate_func: typing.Callable,
log_traj: bool = True):
"""
Compare two configuration wrt the runhistory and return the one which
performs better (or None if the decision is not safe)
Decision strategy to return x as being better than y:
1. x has at least as many runs as y
2. x performs better than y on the intersection of runs on x and y
Implicit assumption:
Challenger was evaluated on the same instance-seed pairs as
incumbent
Parameters
----------
incumbent: Configuration
Current incumbent
challenger: Configuration
Challenger configuration
run_history: RunHistory
Stores all runs we ran so far
aggregate_func: typing.Callable
Aggregate performance across instances
log_traj: bool
Whether to log changes of incumbents in trajectory
Returns
-------
None or better of the two configurations x,y
"""
inc_runs = run_history.get_runs_for_config(incumbent)
chall_runs = run_history.get_runs_for_config(challenger)
to_compare_runs = set(inc_runs).intersection(chall_runs)
# performance on challenger runs
chal_perf = aggregate_func(challenger, run_history, to_compare_runs)
inc_perf = aggregate_func(incumbent, run_history, to_compare_runs)
# Line 15
if chal_perf > inc_perf and len(chall_runs) >= self.minR:
# Incumbent beats challenger
self.logger.debug("Incumbent (%.4f) is better than challenger "
"(%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Line 16
if not set(inc_runs) - set(chall_runs):
# no plateau walks
if chal_perf >= inc_perf:
self.logger.debug(
"Incumbent (%.4f) is at least as good as the "
"challenger (%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Challenger is better than incumbent
# and has at least the same runs as inc
# -> change incumbent
n_samples = len(chall_runs)
self.logger.info(
"Challenger (%.4f) is better than incumbent (%.4f)"
" on %d runs." % (chal_perf, inc_perf, n_samples))
# Show changes in the configuration
params = sorted([(param, incumbent[param], challenger[param])
for param in challenger.keys()])
self.logger.info("Changes in incumbent:")
for param in params:
if param[1] != param[2]:
self.logger.info(" %s : %r -> %r" % (param))
else:
self.logger.debug(" %s remains unchanged: %r" %
(param[0], param[1]))
if log_traj:
self.stats.inc_changed += 1
self.traj_logger.add_entry(train_perf=chal_perf,
incumbent_id=self.stats.inc_changed,
incumbent=challenger)
return challenger
# undecided
return None
def func(x):
return -self.acquisition_function(
[Configuration(self.config_space, vector=x)])