def update_custom_environment_params(self):
"""Try to update null parameters from environment_params_path, or DEFAULT_PARAMS"""
allowed_parameter_keys = [k for k, v in signature(Environment).parameters.items() if v.kind == v.KEYWORD_ONLY]
user_defaults = {}
if (not isinstance(self.environment_params_path, str)) and (self.environment_params_path is not None):
raise TypeError('environment_params_path must be a str, not {}: {}'.format(*type_val(self.environment_params_path)))
try:
user_defaults = read_json(self.environment_params_path)
except TypeError:
if self.environment_params_path is not None:
raise
except FileNotFoundError:
raise
if not isinstance(user_defaults, dict):
raise TypeError('environment_params_path must contain a dict. Received {}: {}'.format(*type_val(user_defaults)))
#################### Check user_defaults ####################
for k, v in user_defaults.items():
if k not in allowed_parameter_keys:
G.warn('\n\t'.join([
'Invalid key ({}) in user-defined default Environment parameter file at "{}". If expected to do something,',
'it really won\'t, so it should be removed or fixed. The following are valid default keys: {}'
]).format(k, self.environment_params_path, allowed_parameter_keys))
elif getattr(self, k) is None:
setattr(self, k, v)
G.debug('Environment kwarg "{}" was set to user default at "{}"'.format(k, self.environment_params_path))
#################### Check Module Default Environment Arguments ####################
for k in allowed_parameter_keys:
if getattr(self, k) is None:
setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
def get_scored_params(experiment_description_path, target_metric):
"""Retrieve the hyperparameters of a completed Experiment, along with an evaluation of its performance
Parameters
----------
experiment_description_path: String
The path to an Experiment's description .json file
target_metric: Tuple
A path denoting the metric to be used. If tuple, the first value should be one of ['oof', 'holdout', 'in_fold'], and the
second value should be the name of a metric supplied in :attr:`environment.Environment.metrics_params`
Returns
-------
all_hyperparameters: Dict
A dict of the hyperparameters used by the Experiment
evaluation: Float
Value of the Experiment's `target_metric`"""
description = read_json(file_path=experiment_description_path)
evaluation = get_path(description['final_evaluations'], target_metric)
all_hyperparameters = description['hyperparameters']
if description['module_name'].lower() == 'keras':
all_hyperparameters['model_init_params'][
'layers'] = consolidate_layers(
all_hyperparameters['model_init_params']['layers'],
class_name_key=False,
separate_args=False)
return (all_hyperparameters, evaluation)
def save_result(self):
self.result = read_json(
f"{self.result_paths['description']}/{self.experiment_id}.json")
make_dirs(self.result_path, exist_ok=True)
with open(f"{self.result_path}/{self.experiment_id}.yml", "w+") as f:
yaml.dump(self.result, f, default_flow_style=False, width=200)
def does_key_exist(self):
"""Check that 1) there is a file corresponding to :attr:`cross_experiment_key.key`, 2) the aforementioned file contains
the key :attr:`key`, and 3) the value of the file at :attr:`key` is a non-empty list
Returns
-------
Boolean"""
if self.cross_experiment_key.exists is True:
records = read_json(F'{self.tested_keys_dir}/{self.cross_experiment_key.key}.json')
for a_hyperparameter_key in records.keys():
if self.key == a_hyperparameter_key:
if isinstance(records[a_hyperparameter_key], list) and len(records[a_hyperparameter_key]) > 0:
self.exists = True
return self.exists
return self.exists
def update_custom_environment_params(self):
"""Try to update null parameters from environment_params_path, or DEFAULT_PARAMS"""
allowed_parameter_keys = [
k for k, v in signature(Environment).parameters.items()
if v.kind == v.KEYWORD_ONLY
]
user_defaults = {}
if (not isinstance(self.environment_params_path, str)) and (
self.environment_params_path is not None):
raise TypeError(
f"Non-str `environment_params_path`: {self.environment_params_path}"
)
try:
user_defaults = read_json(self.environment_params_path)
except TypeError:
if self.environment_params_path is not None:
raise
# If `environment_params_path=None`, no error raised - `user_defaults` continues as {}
except FileNotFoundError:
raise
if not isinstance(user_defaults, dict):
raise TypeError(
"environment_params_path must have dict, not {}".format(
user_defaults))
#################### Check user_defaults ####################
for k, v in user_defaults.items():
if k not in allowed_parameter_keys:
G.warn("\n\t".join([
"Invalid key ({}) in user-defined default Environment parameter file at '{}'. If expected to do something,",
"it really won't, so it should be removed or fixed. The following are valid default keys: {}",
]).format(k, self.environment_params_path,
allowed_parameter_keys))
elif getattr(self, k) is None:
setattr(self, k, v)
G.debug(
f"Environment.`{k}` set to user default: '{self.environment_params_path}'"
)
#################### Check Module Default Environment Arguments ####################
for k in allowed_parameter_keys:
if getattr(self, k) is None:
setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
def get_scored_params(experiment_description_path,
target_metric,
get_description=False):
"""Retrieve the hyperparameters of a completed Experiment, along with its performance evaluation
Parameters
----------
experiment_description_path: String
The path to an Experiment's description .json file
target_metric: Tuple
A path denoting the metric to be used. If tuple, the first value should be one of ['oof',
'holdout', 'in_fold'], and the second value should be the name of a metric supplied in
:attr:`environment.Environment.metrics_params`
get_description: Boolean, default=False
If True, return a tuple of: ((`all_hyperparameters`, `evaluation`), `description`), in which
`description` is the original description dict for the experiment. Else, return a tuple of:
(`all_hyperparameters`, `evaluation`)
Returns
-------
all_hyperparameters: Dict
A dict of the hyperparameters used by the Experiment
evaluation: Float
Value of the Experiment's `target_metric`"""
description = read_json(file_path=experiment_description_path)
evaluation = get_path(description["final_evaluations"], target_metric)
all_hyperparameters = description["hyperparameters"]
if description["module_name"].lower() == "keras":
all_hyperparameters["model_init_params"][
"layers"] = consolidate_layers(
all_hyperparameters["model_init_params"]["layers"],
class_name_key=False)
if get_description:
return ((all_hyperparameters, evaluation), description)
return (all_hyperparameters, evaluation)
def update_custom_environment_params(self):
"""Try to update null parameters from environment_params_path, or DEFAULT_PARAMS"""
allowed_parameter_keys = [
k for k, v in signature(Environment).parameters.items()
if v.kind == v.KEYWORD_ONLY
]
user_defaults = {}
try:
user_defaults = read_json(self.environment_params_path)
except (TypeError, OSError):
# If `environment_params_path=None`, no error raised - `user_defaults` continues as {}
if self.environment_params_path is not None:
raise
if not isinstance(user_defaults, dict):
raise TypeError(
"environment_params_path must have dict, not {}".format(
user_defaults))
#################### Check user_defaults ####################
for k, v in user_defaults.items():
if k not in allowed_parameter_keys:
G.warn(
f"Invalid key ({k}) in user Environment parameters: {self.environment_params_path}"
)
elif getattr(self, k) is None:
setattr(self, k, v)
G.debug(
f"Environment.`{k}` set to user default: '{self.environment_params_path}'"
)
#################### Check Module Default Environment Arguments ####################
for k in allowed_parameter_keys:
if getattr(self, k) is None:
setattr(self, k, self.DEFAULT_PARAMS.get(k, None))
