def _log_incumbent_changes( self, incumbent: Configuration, challenger: Configuration, ) -> None: params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) self.logger.info("Changes in incumbent:") for param in params: if param[1] != param[2]: self.logger.info(" %s : %r -> %r" % param) else: self.logger.debug(" %s remains unchanged: %r", param[0], param[1])
def config_to_html(self, default: Configuration, incumbent: Configuration): """Create HTML-table to compare Configurations. Removes unused parameters. Parameters ---------- default, incumbent: Configurations configurations to be converted Returns ------- table: str HTML-table comparing default and incumbent """ # Remove unused parameters keys = [k for k in default.keys() if default[k] or incumbent[k]] default = [ default[k] if default[k] != None else "inactive" for k in keys ] incumbent = [ incumbent[k] if incumbent[k] != None else "inactive" for k in keys ] table = list(zip(keys, default, incumbent)) # Show first parameters that changed same = [x for x in table if x[1] == x[2]] diff = [x for x in table if x[1] != x[2]] table = [] if len(diff) > 0: table.extend([("-------------- Changed parameters: "\ "--------------", "-----", "-----")]) table.extend(diff) if len(same) > 0: table.extend([("-------------- Unchanged parameters: "\ "--------------", "-----", "-----")]) table.extend(same) keys, table = [k[0] for k in table], [k[1:] for k in table] df = DataFrame(data=table, columns=["Default", "Incumbent"], index=keys) table = df.to_html() return table
def _race_challenger(self, challenger: Configuration, incumbent: Configuration, run_history: RunHistory, aggregate_func: typing.Callable): ''' aggressively race challenger against incumbent Parameters ---------- challenger : Configuration configuration which challenges incumbent incumbent : Configuration best configuration so far run_history : RunHistory stores all runs we ran so far aggregate_func: typing.Callable aggregate performance across instances Returns ------- new_incumbent: Configuration either challenger or incumbent ''' # at least one run of challenger # to increase chall_indx counter first_run = False inc_perf = run_history.get_cost(incumbent) learning_curve = [] self._num_run += 1 self._chall_indx += 1 pc = None for epoch in range(self.max_epochs): status, cost, time, add_info = self.tae_runner.start( config=challenger, instance=None, seed=0, cutoff=2**32 - 1, instance_specific=None, pc=pc) try: pc = add_info["model"] except KeyError: # model building failed, e.g. because of nan break learning_curve.append(cost) if len(self.learning_curves) > 10 and epoch > self.max_epochs / 4: seen_curves = np.array(self.learning_curves)[:, epoch] if cost > np.median(seen_curves): self.logger.info("Abort run (%f vs %f)" % (cost, np.median(seen_curves))) break # delete model in runhistory to be more memory efficient chall_id = run_history.config_ids[challenger] runkey = RunKey(chall_id, None, 0) runvalue = run_history.data[runkey] try: del runvalue.additional_info["model"] except KeyError: pass if epoch == self.max_epochs - 1: self.learning_curves.append(learning_curve) chal_perf = cost if cost < inc_perf: self.logger.info( "Challenger (%.4f) is better than incumbent (%.4f)" % (chal_perf, inc_perf)) # Show changes in the configuration params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) self.logger.info("Changes in incumbent:") for param in params: if param[1] != param[2]: self.logger.info(" %s : %r -> %r" % (param)) else: self.logger.debug(" %s remains unchanged: %r" % (param[0], param[1])) incumbent = challenger self.stats.inc_changed += 1 self.traj_logger.add_entry(train_perf=chal_perf, incumbent_id=self.stats.inc_changed, incumbent=challenger) else: self.logger.debug( "Incumbent (%.4f) is better than challenger (%.4f)" % (inc_perf, chal_perf)) return incumbent