def _kwargs(self, object_type, kwargs, prefix="default_kwargs"):
default_kwargs_getter = getattr(self,
"{}_{}".format(prefix,
object_type), None)
if default_kwargs_getter is None:
if len(kwargs) == 0:
return None
final_kwargs = emag_utils.merge_two_dicts({}, kwargs)
else:
final_kwargs = emag_utils.merge_two_dicts(default_kwargs_getter(),
kwargs)
return {k: v() for k, v in final_kwargs.items()}
def save_config_files(config_folder, dict_of_yamls, resume_training,
reproduce_results, latest_epochs):
makedir_if_not_there(config_folder)
new_dir = None
for k, v in dict_of_yamls.items():
k_split = k.split('/')
config_category_name = k_split[-2]
if not config_category_name.startswith('config_'):
config_category_name = os.path.splitext(k_split[-1])[0]
fname = os.path.join(config_folder, '%s.yaml' % config_category_name)
if not resume_training:
if os.path.isfile(fname):
v = emag_utils.merge_two_dicts(
load_yaml(fname),
v,
max_merge_depth=0 if reproduce_results else float('inf'))
write_yaml(fname, v, 'w')
else:
curr_yaml = load_yaml(fname)
yaml_diff = {}
for k2, v2 in v.items():
if (k2 not in curr_yaml) or (v2 != curr_yaml[k2]):
yaml_diff[k2] = v2
if yaml_diff != {}:
new_dir = os.path.join(
config_folder, 'resume_training_config_diffs_' +
'_'.join([str(epoch) for epoch in latest_epochs]))
makedir_if_not_there(new_dir)
fname = os.path.join(new_dir, '%s.yaml' % config_category_name)
write_yaml(fname, yaml_diff, 'a')
def save_config_files(config_folder, dict_of_yamls, resume_training,
latest_epochs):
makedir_if_not_there(config_folder)
new_dir = None
existing_config_diff_folders, _ = get_sorted_config_diff_folders(
config_folder)
for config_name, config_dict in dict_of_yamls.items():
fname = os.path.join(config_folder, '%s.yaml' % config_name)
if not resume_training:
write_yaml(fname, config_dict, 'w')
else:
curr_yaml = load_yaml(fname)
for config_diff_folder in existing_config_diff_folders:
config_diff = os.path.join(config_diff_folder,
'%s.yaml' % config_name)
if os.path.isfile(config_diff):
curr_yaml = emag_utils.merge_two_dicts(
curr_yaml,
load_yaml(config_diff),
max_merge_depth=float('inf'))
yaml_diff = {}
for k, v in config_dict.items():
if (k not in curr_yaml) or (v != curr_yaml[k]):
yaml_diff[k] = v
if yaml_diff != {}:
new_dir = os.path.join(
config_folder, CONFIG_DIFF_BASE_FOLDER_NAME +
'_'.join([str(epoch) for epoch in latest_epochs]))
makedir_if_not_there(new_dir)
fname = os.path.join(new_dir, '%s.yaml' % config_name)
write_yaml(fname, yaml_diff, 'a')
def get_api_parser(self, args):
api_parser_kwargs = {
"args": args,
"pytorch_getter": self.pytorch_getter,
"global_db_path": self.global_db_path
}
if args.api_parser:
AP, AP_params = self.pytorch_getter.get('api_parser',
yaml_dict=args.api_parser,
return_uninitialized=True)
AP_params = emag_utils.merge_two_dicts(api_parser_kwargs,
AP_params)
AP = AP(**AP_params)
else:
class_name = "API" + c_f.first_key_of_dict(args.trainer)
try:
AP = self.pytorch_getter.get('api_parser',
class_name=class_name,
params=api_parser_kwargs)
except AttributeError:
logging.warning(
"Couldn't find api_parser named {}. Using BaseAPIParser instead."
.format(class_name))
AP = self.pytorch_getter.get('api_parser',
class_name="BaseAPIParser",
params=api_parser_kwargs)
return AP
def _create_general(self, tester_type, plots_folder, **kwargs):
tester, tester_params = self.getter.get("tester",
yaml_dict=tester_type,
return_uninitialized=True)
tester_params = copy.deepcopy(tester_params)
tester_params["accuracy_calculator"] = self.getter.get(
"accuracy_calculator",
yaml_dict=tester_params["accuracy_calculator"])
if tester_params.get("visualizer", None):
tester_params["visualizer"] = self.getter.get(
"visualizer", yaml_dict=tester_params["visualizer"])
tester_params["visualizer_hook"] = self.visualizer_hook(
plots_folder)
tester_params = emag_utils.merge_two_dicts(tester_params, kwargs)
return tester(**tester_params)
def _create_general(self, trainer_type, **kwargs):
trainer, trainer_params = self.getter.get("trainer",
yaml_dict=trainer_type,
return_uninitialized=True)
trainer_params = emag_utils.merge_two_dicts(trainer_params, kwargs)
return trainer(**trainer_params)
def create_hook_container(self, hook_container_type, **kwargs):
hooks, hooks_params = self.getter.get("hook_container",
yaml_dict=hook_container_type,
return_uninitialized=True)
hooks_params = emag_utils.merge_two_dicts(hooks_params, kwargs)
return hooks(**hooks_params)
def merge_key_specific_kwargs(self, kwargs, kwargs_for_this_name, key):
key_specific_kwargs = self.key_specific_kwargs(key)
x = emag_utils.merge_two_dicts(kwargs, key_specific_kwargs)
return emag_utils.merge_two_dicts(x, kwargs_for_this_name)