def add_complex_type_lookup_entry(self, path, key, value, hashed_value): """Add lookup entry in `key_attribute_lookup_dir` for a complex-typed parameter, linking the parameter `key`, its `value`, and its `hashed_value` Parameters ---------- path: Tuple The path of keys that leads to `key` key: Str The parameter name value: * The value of the parameter `key` hashed_value: Str The hash produced for `value`""" # TODO: Combine `path` and `key` to produce actual filepaths shelve_params = ['model_initializer', 'cross_validation_type'] if isclass(value) or (key in shelve_params): with shelve.open(os.path.join(self.key_attribute_lookup_dir, F'{key}'), flag='c') as shelf: # NOTE: When reading from shelve file, DO NOT add the ".db" file extension shelf[hashed_value] = value elif isinstance(value, pd.DataFrame): os.makedirs(os.path.join(self.key_attribute_lookup_dir, key), exist_ok=True) value.to_csv(os.path.join(self.key_attribute_lookup_dir, key, F'{hashed_value}.csv'), index=False) else: # Possible types: partial, function, *other add_to_json( file_path=os.path.join(self.key_attribute_lookup_dir, F'{key}.json'), data_to_add=getsource(value), key=hashed_value, condition=lambda _: hashed_value not in _.keys(), default={}, )
def add_complex_type_lookup_entry(self, path, key, value, hashed_value): """Add lookup entry in `lookup_dir` for a complex-typed parameter, linking the parameter `key`, its `value`, and its `hashed_value` Parameters ---------- path: Tuple The path of keys that leads to `key` key: Str The parameter name value: * The value of the parameter `key` hashed_value: Str The hash produced for `value`""" shelve_params = ["model_initializer", "cv_type"] lookup_path = partial(os.path.join, self.lookup_dir, *[f"{_}" for _ in path]) if isclass(value) or (key in shelve_params): make_dirs(lookup_path(), exist_ok=True) with shelve.open(lookup_path(f"{key}"), flag="c") as s: # NOTE: When reading from shelve file, DO NOT add the ".db" file extension s[hashed_value] = value elif isinstance(value, pd.DataFrame): make_dirs(lookup_path(key), exist_ok=True) value.to_csv(lookup_path(key, f"{hashed_value}.csv"), index=False) else: # Possible types: partial, function, *other add_to_json( file_path=lookup_path(f"{key}.json"), data_to_add=getsource(value), key=hashed_value, condition=lambda _: hashed_value not in _.keys(), default={}, )
def save_result(self): """Save the cross-experiment, and hyperparameter keys, and update their tested keys entries""" self.cross_experiment_key.save_key() self.hyperparameter_key.save_key() add_to_json( file_path= F'{self.hyperparameter_key.tested_keys_dir}/{self.cross_experiment_key.key}.json', data_to_add=self.experiment_id, key=self.hyperparameter_key.key, condition=lambda _: self.hyperparameter_key.key in _.keys(), append_value=True)
def save_key(self): """Create an entry in the dict contained in the file at :attr:`cross_experiment_key.key`, whose key is :attr:`key`, and whose value is an empty list if :attr:`exists` is False""" if not self.exists: if self.cross_experiment_key.exists is False: _err = "Cannot save hyperparameter_key: '{}', before cross_experiment_key '{}'" raise ValueError(_err.format(self.key, self.cross_experiment_key.key)) key_path = f"{self.tested_keys_dir}/{self.cross_experiment_key.key}.json" add_to_json(key_path, [], key=self.key, condition=lambda _: self.key not in _.keys()) self.exists = True G.log(f'Saved {self.key_type}_key: "{self.key}"', 4) else: G.log(f'{self.key_type}_key "{self.key}" already exists - Skipped saving', 4)
def save_key(self): """Create an entry in the dict corresponding to the file at :attr:`cross_experiment_key.key`, whose key is :attr:`key`, and whose value is an empty list if :attr:`exists` is False""" if not self.exists: if self.cross_experiment_key.exists is False: raise ValueError('Cannot save hyperparameter_key: "{}", before cross_experiment_key "{}" has been saved'.format( self.key, self.cross_experiment_key.key )) key_path = F'{self.tested_keys_dir}/{self.cross_experiment_key.key}.json' add_to_json(file_path=key_path, data_to_add=[], key=self.key, condition=lambda _: self.key not in _.keys()) self.exists = True G.log(F'Saved {self.key_type}_key: "{self.key}"') else: G.log(F'{self.key_type}_key "{self.key}" already exists - Skipped saving')
def add_complex_type_lookup_entry(self, path, key, value, hashed_value): """Add lookup entry in `lookup_dir` for a complex-typed parameter, linking the parameter `key`, its `value`, and its `hashed_value` Parameters ---------- path: Tuple The path of keys that leads to `key` key: Str The parameter name value: * The value of the parameter `key` hashed_value: Str The hash produced for `value`""" shelve_params = ["model_initializer", "cv_type"] lookup_path = partial(os.path.join, self.lookup_dir, *[f"{_}" for _ in path]) if isclass(value) or (key in shelve_params): make_dirs(lookup_path(), exist_ok=True) with shelve.open(lookup_path(f"{key}"), flag="c") as s: # NOTE: When reading from shelve file, DO NOT add the ".db" file extension try: s[hashed_value] = value except PicklingError: # "is not the same object" error can be raised due to `Mirror`/`TranslateTrace` # Instead of saving the object that raised the error, save `getsourcelines` # Source lines of traced object are identical to those of its un-traced original s[hashed_value] = getsourcelines(value) except Exception: raise elif isinstance(value, pd.DataFrame): make_dirs(lookup_path(key), exist_ok=True) value.to_csv(lookup_path(key, f"{hashed_value}.csv"), index=False) else: # Possible types: partial, function, *other add_to_json( file_path=lookup_path(f"{key}.json"), data_to_add=getsource(value), key=hashed_value, condition=lambda _: hashed_value not in _.keys(), default={}, )