def update_custom_environment_params(self): """Try to update null parameters from environment_params_path, or DEFAULT_PARAMS""" allowed_parameter_keys = [k for k, v in signature(Environment).parameters.items() if v.kind == v.KEYWORD_ONLY] user_defaults = {} if (not isinstance(self.environment_params_path, str)) and (self.environment_params_path is not None): raise TypeError('environment_params_path must be a str, not {}: {}'.format(*type_val(self.environment_params_path))) try: user_defaults = read_json(self.environment_params_path) except TypeError: if self.environment_params_path is not None: raise except FileNotFoundError: raise if not isinstance(user_defaults, dict): raise TypeError('environment_params_path must contain a dict. Received {}: {}'.format(*type_val(user_defaults))) #################### Check user_defaults #################### for k, v in user_defaults.items(): if k not in allowed_parameter_keys: G.warn('\n\t'.join([ 'Invalid key ({}) in user-defined default Environment parameter file at "{}". If expected to do something,', 'it really won\'t, so it should be removed or fixed. The following are valid default keys: {}' ]).format(k, self.environment_params_path, allowed_parameter_keys)) elif getattr(self, k) is None: setattr(self, k, v) G.debug('Environment kwarg "{}" was set to user default at "{}"'.format(k, self.environment_params_path)) #################### Check Module Default Environment Arguments #################### for k in allowed_parameter_keys: if getattr(self, k) is None: setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
def get_scored_params(experiment_description_path, target_metric): """Retrieve the hyperparameters of a completed Experiment, along with an evaluation of its performance Parameters ---------- experiment_description_path: String The path to an Experiment's description .json file target_metric: Tuple A path denoting the metric to be used. If tuple, the first value should be one of ['oof', 'holdout', 'in_fold'], and the second value should be the name of a metric supplied in :attr:`environment.Environment.metrics_params` Returns ------- all_hyperparameters: Dict A dict of the hyperparameters used by the Experiment evaluation: Float Value of the Experiment's `target_metric`""" description = read_json(file_path=experiment_description_path) evaluation = get_path(description['final_evaluations'], target_metric) all_hyperparameters = description['hyperparameters'] if description['module_name'].lower() == 'keras': all_hyperparameters['model_init_params'][ 'layers'] = consolidate_layers( all_hyperparameters['model_init_params']['layers'], class_name_key=False, separate_args=False) return (all_hyperparameters, evaluation)
def save_result(self): self.result = read_json( f"{self.result_paths['description']}/{self.experiment_id}.json") make_dirs(self.result_path, exist_ok=True) with open(f"{self.result_path}/{self.experiment_id}.yml", "w+") as f: yaml.dump(self.result, f, default_flow_style=False, width=200)
def does_key_exist(self): """Check that 1) there is a file corresponding to :attr:`cross_experiment_key.key`, 2) the aforementioned file contains the key :attr:`key`, and 3) the value of the file at :attr:`key` is a non-empty list Returns ------- Boolean""" if self.cross_experiment_key.exists is True: records = read_json(F'{self.tested_keys_dir}/{self.cross_experiment_key.key}.json') for a_hyperparameter_key in records.keys(): if self.key == a_hyperparameter_key: if isinstance(records[a_hyperparameter_key], list) and len(records[a_hyperparameter_key]) > 0: self.exists = True return self.exists return self.exists
def update_custom_environment_params(self): """Try to update null parameters from environment_params_path, or DEFAULT_PARAMS""" allowed_parameter_keys = [ k for k, v in signature(Environment).parameters.items() if v.kind == v.KEYWORD_ONLY ] user_defaults = {} if (not isinstance(self.environment_params_path, str)) and ( self.environment_params_path is not None): raise TypeError( f"Non-str `environment_params_path`: {self.environment_params_path}" ) try: user_defaults = read_json(self.environment_params_path) except TypeError: if self.environment_params_path is not None: raise # If `environment_params_path=None`, no error raised - `user_defaults` continues as {} except FileNotFoundError: raise if not isinstance(user_defaults, dict): raise TypeError( "environment_params_path must have dict, not {}".format( user_defaults)) #################### Check user_defaults #################### for k, v in user_defaults.items(): if k not in allowed_parameter_keys: G.warn("\n\t".join([ "Invalid key ({}) in user-defined default Environment parameter file at '{}'. If expected to do something,", "it really won't, so it should be removed or fixed. The following are valid default keys: {}", ]).format(k, self.environment_params_path, allowed_parameter_keys)) elif getattr(self, k) is None: setattr(self, k, v) G.debug( f"Environment.`{k}` set to user default: '{self.environment_params_path}'" ) #################### Check Module Default Environment Arguments #################### for k in allowed_parameter_keys: if getattr(self, k) is None: setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
def get_scored_params(experiment_description_path, target_metric, get_description=False): """Retrieve the hyperparameters of a completed Experiment, along with its performance evaluation Parameters ---------- experiment_description_path: String The path to an Experiment's description .json file target_metric: Tuple A path denoting the metric to be used. If tuple, the first value should be one of ['oof', 'holdout', 'in_fold'], and the second value should be the name of a metric supplied in :attr:`environment.Environment.metrics_params` get_description: Boolean, default=False If True, return a tuple of: ((`all_hyperparameters`, `evaluation`), `description`), in which `description` is the original description dict for the experiment. Else, return a tuple of: (`all_hyperparameters`, `evaluation`) Returns ------- all_hyperparameters: Dict A dict of the hyperparameters used by the Experiment evaluation: Float Value of the Experiment's `target_metric`""" description = read_json(file_path=experiment_description_path) evaluation = get_path(description["final_evaluations"], target_metric) all_hyperparameters = description["hyperparameters"] if description["module_name"].lower() == "keras": all_hyperparameters["model_init_params"][ "layers"] = consolidate_layers( all_hyperparameters["model_init_params"]["layers"], class_name_key=False) if get_description: return ((all_hyperparameters, evaluation), description) return (all_hyperparameters, evaluation)
def update_custom_environment_params(self): """Try to update null parameters from environment_params_path, or DEFAULT_PARAMS""" allowed_parameter_keys = [ k for k, v in signature(Environment).parameters.items() if v.kind == v.KEYWORD_ONLY ] user_defaults = {} try: user_defaults = read_json(self.environment_params_path) except (TypeError, OSError): # If `environment_params_path=None`, no error raised - `user_defaults` continues as {} if self.environment_params_path is not None: raise if not isinstance(user_defaults, dict): raise TypeError( "environment_params_path must have dict, not {}".format( user_defaults)) #################### Check user_defaults #################### for k, v in user_defaults.items(): if k not in allowed_parameter_keys: G.warn( f"Invalid key ({k}) in user Environment parameters: {self.environment_params_path}" ) elif getattr(self, k) is None: setattr(self, k, v) G.debug( f"Environment.`{k}` set to user default: '{self.environment_params_path}'" ) #################### Check Module Default Environment Arguments #################### for k in allowed_parameter_keys: if getattr(self, k) is None: setattr(self, k, self.DEFAULT_PARAMS.get(k, None))