def _get_scored_params(self): """For all :attr:`experiment_ids`, add a tuple of the Experiment's hyperparameters, and its :attr:`target_metric` value""" for _id in self.experiment_ids: self.hyperparameters_and_scores.append( get_scored_params( '{}/{}.json'.format(self.descriptions_dir, _id), self.target_metric) + (_id, ))
def _get_scored_params(self): """For all :attr:`experiment_ids`, add a tuple of the Experiment's hyperparameters, and its :attr:`target_metric` value""" for _id in self.experiment_ids: vals = get_scored_params(f"{self.descriptions_dir}/{_id}.json", self.target_metric) self.hyperparameters_and_scores.append(vals + (_id, ))
def _get_scored_params(self): """For all :attr:`experiment_ids`, add a tuple of the Experiment's hyperparameters, and its :attr:`target_metric` value""" for _id in self.experiment_ids: # TODO: Receive `description` from `get_scored_params` and extract whatever value is required by :attr:`sort` vals = get_scored_params(f"{self.descriptions_dir}/{_id}.json", self.target_metric) self.hyperparameters_and_scores.append(vals + (_id, ))
def find(self): """Execute full result-finding workflow, populating :attr:`similar_experiments` See Also -------- :func:`update_match_status` Used to decorate "does_match..." methods in order to keep a detailed record of the full pool of candidate Experiments in :attr:`match_status`. Aside from being used to compile the list of finalist :attr:`similar_experiments`, :attr:`match_status` is helpful for debugging purposes, specifically figuring out which aspects of a candidate are incompatible with the template :meth:`does_match_feature_engineer` Performs special functionality beyond that of the other "does_match..." methods, namely providing an updated "feature_engineer" value for compatible candidates to use. Specifics are documented in :meth:`does_match_feature_engineer`""" for exp_id in self.experiment_ids: description_path = f"{self.descriptions_dir}/{exp_id}.json" # TODO: Get `description` from `get_scored_params` - Take whatever value `sort` needs params, score = get_scored_params(description_path, self.target_metric) #################### Match Init Params #################### self.does_match_init_params_space(exp_id, params["model_init_params"], score) multi_targets = [("model_init_params", "compile_params", "optimizer")] if self.module_name == "keras" and multi_targets[0] in self.space.names(): self.does_match_init_params_guidelines_multi( exp_id, params["model_init_params"], score, multi_targets[0][1:] ) else: self.does_match_init_params_guidelines(exp_id, params["model_init_params"], score) #################### Match Extra Params #################### self.does_match_extra_params_space(exp_id, params["model_extra_params"], score) self.does_match_extra_params_guidelines(exp_id, params["model_extra_params"], score) #################### Match Feature Engineer #################### # NOTE: Matching "feature_engineer" is critically different from the other "does_match" # methods. `does_match_feature_engineer` builds on the straight-forward compatibility # checks of the others by returning an updated "feature_engineer" for the candidate # if compatible. See :meth:`does_match_feature_engineer` for details params["feature_engineer"] = self.does_match_feature_engineer( exp_id, params["feature_engineer"], score ) # Since updated "feature_engineer" is saved in `params`, clean up `match_status` entry if self.match_status[exp_id]["does_match_feature_engineer"] is not False: self.match_status[exp_id]["does_match_feature_engineer"] = True #################### Determine Overall Match #################### if all(v for k, v in self.match_status[exp_id].items() if k.startswith("does_match")): self.similar_experiments.append((params, score, exp_id)) G.debug_( "Matching Experiments: {} (Candidates: {})".format( len(self.similar_experiments), len(self.experiment_ids) ) )