def load_config(path: str) -> ConfigType: if path is None: return {} elif not isinstance(path, str): # TODO: unnecessary raise TypeError(path) elif not os.path.exists(path): return Module() elif os.path.isdir(path): _dict: ConfigType = Module() for root, dirs, files in os.walk(path): for _file in files: name, ext = os.path.splitext(_file) file_path = os.path.normpath(os.path.join(root, _file)) assert name not in _dict.keys( ), f'filename conflicts: {file_path}' if ext in ['.yml', '.yaml', 'json']: _dict.update(Config.load_config(file_path)) return _dict elif os.path.isfile(path): name, ext = os.path.splitext(os.path.split(path)[1]) if ext in ['.yml', 'yaml', 'json']: with open(path, 'r', encoding='utf-8') as f: _dict: dict[str, Union[Any, dict[str, Any]]] = {} if ext == 'json': _dict = json.load(f.read()) else: _dict = yaml.load(f.read(), Loader=yaml.FullLoader) return Module(**{name: Config.organize_config_file(_dict)}) else: return Module() else: raise Exception(f'unknown: {path}')
def combine( self, keys: list[str] = ['package', 'user', 'project']) -> ConfigType: config = Module() for key in keys: if key in self.config_dict.keys(): config.update(self.config_dict[key]) return config
def initialize_folder(self): _dict = Module(self.__dict__) _dict.__delattr__('folder_path') imagenet = ImageNet(**_dict) class_dict: dict = {} json_path = os.path.normpath(os.path.join(root_dir, 'data', self.name, 'class_dict.json')) with open(json_path, 'r', encoding='utf-8') as f: class_dict: dict = json.load(f) imagenet.sample(child_name=self.name, class_dict=class_dict)
def organize_config_file( _dict: dict[str, Union[Any, dict[str, Any]]]) -> ConfigFileType: module = Module() for key, value in _dict.items(): if isinstance(value, dict): value = Param(value) module[key] = value # TODO: Shall we Param(value) ? return module
def get_config(self, dataset_name: str, config: ConfigType = None, **kwargs) -> Param[str, Module[str, Any]]: config = config if config is not None else Param( self.get_full_config(), default=Module()) # remove dataset_name Param for file_name, file_value in config.items(): if not isinstance(file_value, Module) and not isinstance( file_value, dict): # TODO: remove the latter condition? continue if isinstance(file_value, Param): config[file_name] = file_value[dataset_name] continue for param_name, param_value in file_value.items(): if isinstance(param_value, Param): config[file_name][param_name] = param_value[dataset_name] # else: # raise TypeError(f'{type(param_value)=} {param_value=}') config.update(kwargs) return config
def initialize_folder(self): _dict = Module(self.__dict__) _dict.__delattr__('folder_path') vggface2 = VGGface2(**_dict) vggface2.sample(child_name=self.name, sample_num=self.num_classes)