def patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]:
patch = load_yml(patch_path)
experiment_base = patch['experiment_name']
out_folder = patch['yml_output_folder']
mkdir(out_folder)
for key, values in patch['run_cfg'].items():
for value in values:
cfg = ConfigLoader().from_file(cfg_path, suppress_print=True)
broadcast = key.split('/')
broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None
if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1:
layer_key = key.split('/')[-1]
for i in [int(j) for j in broadcast]:
b_key = f'layers/{i}/{layer_key}'
cfg[b_key] = value
assert cfg[b_key] == value
else:
cfg[key] = value
assert cfg[key] == value
param_name = key.replace('/','_').split('_')
param_name = ''.join([s[0] for s in param_name])
str_value = str(value).replace('.', '_')
ex_uuid = uuid.uuid4().hex[0:6]
experiment_name = f'{experiment_base}_{ex_uuid}_{param_name}={str_value}'
cfg['model_path'] = f'trained_models/{experiment_name}'
out_path = f'{out_folder}/{experiment_name}.yml'
logging.info(f'Saving param: {key} value: {value} to {out_path}')
save_yml(cfg.env, out_path)
def test_array(self):
document = """
l:
- list: str
attr: 1
- test: nr
attr: 2
"""
loader = ConfigLoader().from_string(document)
self.assertEqual(loader['l/0/attr'], 1)
self.assertEqual(loader['l/1/attr'], 2)
loader['l/0/attr'] = 3
loader['l/1/attr'] = 4
self.assertEqual(loader['l/0/attr'], 3)
self.assertEqual(loader['l/1/attr'], 4)
def test_config_loader_from_str(self):
document = """
a: 1
b:
c: 3
d: 4
"""
loader = ConfigLoader().from_string(document)
self.assertEqual(loader['b/c'], 3)
self.assertEqual(loader['a'], 1)
loader['a'] = 'bär'
self.assertEqual(loader['a'], 'bär')
self.assertEqual(loader['b']['c'], 3)
self.assertEqual(loader['b']['d'], 4)
self.assertEqual(loader['b', 'c'], 3)
def permute_patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]:
patch = load_yml(patch_path)
experiment_base = patch['experiment_name']
out_folder = patch['yml_output_folder']
mkdir(out_folder)
keys = []
values = []
for key, vals in patch['run_cfg'].items():
keys += [key]
values.append(vals)
permutations = list(product(*values))
num_permutations = len(permutations)
logging.info(f"Generating {num_permutations} Permuted Configurations")
# iterate over all permutation tuples
for tple in permutations:
fn_prefix = ''
cfg = ConfigLoader().from_file(cfg_path, suppress_print=True)
for id_k, key in enumerate(keys):
broadcast = key.split('/')
broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None
if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1:
layer_key = key.split('/')[-1]
for i in [int(j) for j in broadcast]:
b_key = f'layers/{i}/{layer_key}'
cfg[b_key] = tple[id_k]
assert cfg[b_key] == tple[id_k]
else:
cfg[key] = tple[id_k]
assert cfg[key] == tple[id_k]
param_name = key.replace('/','_').split('_')
param_name = ''.join([s[0] for s in param_name])
str_value = str(tple[id_k]).replace('.', '_')
fn_prefix += f'_{param_name}={str_value}'
ex_uuid = uuid.uuid4().hex[0:6]
experiment_name = f'{experiment_base}_{ex_uuid}_{fn_prefix}'
cfg['model_path'] = f'trained_models/{experiment_name}'
out_path = f'{out_folder}/{experiment_name}.yml'
logging.info(f'Saving tuple: {tple} to {out_path}')
save_yml(cfg.env, out_path)
def test_config_loader_from_file(self):
loader = ConfigLoader().from_file('yaml/env_template.yml')
self.assertEqual(loader['datasets/cifar10/path'], '<path>')
def test_cfg_to_network(self):
rcfg = ConfigLoader().from_file('src/yaml/nets/net_template.yml')
gcfg = ConfigLoader().from_file('src/yaml/env.yml')
net = cfg_to_network(gcfg, rcfg)
print(net)
def test_save(self):
cfg = ConfigLoader().from_file('yaml/env_template.yml')
net = AutoencoderNet(cfg['datasets/cifar10/path'])
layers: List[LayerTrainingDefinition] = net.layer_configs
net.save_layer(layers[0].model, self.file)
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('--cfg', type=file_path, required=True,
help='The main config yaml file.')
parser.add_argument('--env', type=file_path, default='src/yaml/env.yml',
help='The environment yaml file.')
args = parser.parse_args()
run_cfg_path = args.cfg
env_cfg_path = args.env
# Load configs
env_cfg = ConfigLoader().from_file(env_cfg_path)
run_cfg = ConfigLoader().from_file(run_cfg_path)
# copy the configs to dir for documentation / remembering
rcfg_fn = os.path.split(run_cfg_path)[-1]
ecfg_fn = os.path.split(env_cfg_path)[-1]
model_path = run_cfg['model_path']
if not os.path.exists(model_path):
os.makedirs(model_path)
logging.info(f'Created model path: {model_path}')
else:
logging.warning(f'Model path exists already: {model_path}')
logging.info(f'Copy {run_cfg_path} to {model_path}/{rcfg_fn}')
logging.info(f'Copy {env_cfg_path} to {model_path}/{ecfg_fn}')