def load_config(path: str) -> ConfigType:
if path is None:
return {}
elif not isinstance(path, str): # TODO: unnecessary
raise TypeError(path)
elif not os.path.exists(path):
return Module()
elif os.path.isdir(path):
_dict: ConfigType = Module()
for root, dirs, files in os.walk(path):
for _file in files:
name, ext = os.path.splitext(_file)
file_path = os.path.normpath(os.path.join(root, _file))
assert name not in _dict.keys(
), f'filename conflicts: {file_path}'
if ext in ['.yml', '.yaml', 'json']:
_dict.update(Config.load_config(file_path))
return _dict
elif os.path.isfile(path):
name, ext = os.path.splitext(os.path.split(path)[1])
if ext in ['.yml', 'yaml', 'json']:
with open(path, 'r', encoding='utf-8') as f:
_dict: dict[str, Union[Any, dict[str, Any]]] = {}
if ext == 'json':
_dict = json.load(f.read())
else:
_dict = yaml.load(f.read(), Loader=yaml.FullLoader)
return Module(**{name: Config.organize_config_file(_dict)})
else:
return Module()
else:
raise Exception(f'unknown: {path}')
def combine(
self,
keys: list[str] = ['package', 'user', 'project']) -> ConfigType:
config = Module()
for key in keys:
if key in self.config_dict.keys():
config.update(self.config_dict[key])
return config
def initialize_folder(self):
_dict = Module(self.__dict__)
_dict.__delattr__('folder_path')
imagenet = ImageNet(**_dict)
class_dict: dict = {}
json_path = os.path.normpath(os.path.join(root_dir, 'data', self.name, 'class_dict.json'))
with open(json_path, 'r', encoding='utf-8') as f:
class_dict: dict = json.load(f)
imagenet.sample(child_name=self.name, class_dict=class_dict)
def organize_config_file(
_dict: dict[str, Union[Any, dict[str, Any]]]) -> ConfigFileType:
module = Module()
for key, value in _dict.items():
if isinstance(value, dict):
value = Param(value)
module[key] = value # TODO: Shall we Param(value) ?
return module
def get_config(self,
dataset_name: str,
config: ConfigType = None,
**kwargs) -> Param[str, Module[str, Any]]:
config = config if config is not None else Param(
self.get_full_config(), default=Module())
# remove dataset_name Param
for file_name, file_value in config.items():
if not isinstance(file_value, Module) and not isinstance(
file_value, dict):
# TODO: remove the latter condition?
continue
if isinstance(file_value, Param):
config[file_name] = file_value[dataset_name]
continue
for param_name, param_value in file_value.items():
if isinstance(param_value, Param):
config[file_name][param_name] = param_value[dataset_name]
# else:
# raise TypeError(f'{type(param_value)=} {param_value=}')
config.update(kwargs)
return config
def initialize_folder(self):
_dict = Module(self.__dict__)
_dict.__delattr__('folder_path')
vggface2 = VGGface2(**_dict)
vggface2.sample(child_name=self.name, sample_num=self.num_classes)
