def patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]:
patch = load_yml(patch_path)
experiment_base = patch['experiment_name']
out_folder = patch['yml_output_folder']
mkdir(out_folder)
for key, values in patch['run_cfg'].items():
for value in values:
cfg = ConfigLoader().from_file(cfg_path, suppress_print=True)
broadcast = key.split('/')
broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None
if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1:
layer_key = key.split('/')[-1]
for i in [int(j) for j in broadcast]:
b_key = f'layers/{i}/{layer_key}'
cfg[b_key] = value
assert cfg[b_key] == value
else:
cfg[key] = value
assert cfg[key] == value
param_name = key.replace('/','_').split('_')
param_name = ''.join([s[0] for s in param_name])
str_value = str(value).replace('.', '_')
ex_uuid = uuid.uuid4().hex[0:6]
experiment_name = f'{experiment_base}_{ex_uuid}_{param_name}={str_value}'
cfg['model_path'] = f'trained_models/{experiment_name}'
out_path = f'{out_folder}/{experiment_name}.yml'
logging.info(f'Saving param: {key} value: {value} to {out_path}')
save_yml(cfg.env, out_path)
def test_array(self):
document = """
l:
- list: str
attr: 1
- test: nr
attr: 2
"""
loader = ConfigLoader().from_string(document)
self.assertEqual(loader['l/0/attr'], 1)
self.assertEqual(loader['l/1/attr'], 2)
loader['l/0/attr'] = 3
loader['l/1/attr'] = 4
self.assertEqual(loader['l/0/attr'], 3)
self.assertEqual(loader['l/1/attr'], 4)
def test_config_loader_from_str(self):
document = """
a: 1
b:
c: 3
d: 4
"""
loader = ConfigLoader().from_string(document)
self.assertEqual(loader['b/c'], 3)
self.assertEqual(loader['a'], 1)
loader['a'] = 'bär'
self.assertEqual(loader['a'], 'bär')
self.assertEqual(loader['b']['c'], 3)
self.assertEqual(loader['b']['d'], 4)
self.assertEqual(loader['b', 'c'], 3)
def permute_patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]:
patch = load_yml(patch_path)
experiment_base = patch['experiment_name']
out_folder = patch['yml_output_folder']
mkdir(out_folder)
keys = []
values = []
for key, vals in patch['run_cfg'].items():
keys += [key]
values.append(vals)
permutations = list(product(*values))
num_permutations = len(permutations)
logging.info(f"Generating {num_permutations} Permuted Configurations")
# iterate over all permutation tuples
for tple in permutations:
fn_prefix = ''
cfg = ConfigLoader().from_file(cfg_path, suppress_print=True)
for id_k, key in enumerate(keys):
broadcast = key.split('/')
broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None
if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1:
layer_key = key.split('/')[-1]
for i in [int(j) for j in broadcast]:
b_key = f'layers/{i}/{layer_key}'
cfg[b_key] = tple[id_k]
assert cfg[b_key] == tple[id_k]
else:
cfg[key] = tple[id_k]
assert cfg[key] == tple[id_k]
param_name = key.replace('/','_').split('_')
param_name = ''.join([s[0] for s in param_name])
str_value = str(tple[id_k]).replace('.', '_')
fn_prefix += f'_{param_name}={str_value}'
ex_uuid = uuid.uuid4().hex[0:6]
experiment_name = f'{experiment_base}_{ex_uuid}_{fn_prefix}'
cfg['model_path'] = f'trained_models/{experiment_name}'
out_path = f'{out_folder}/{experiment_name}.yml'
logging.info(f'Saving tuple: {tple} to {out_path}')
save_yml(cfg.env, out_path)
def cfg_to_network(gcfg: ConfigLoader, rcfg: ConfigLoader) \
-> List[LayerTrainingDefinition]:
r"""
This is the main network assembly function. It takes a config
from a configloader (a yaml file). Assembles a network stack
as list of LayerTrainingDefintions.
Returns:
List[LayerTrainingDefinition]
"""
# params from run config
num_layers = len(rcfg['layers'])
weight_decay = rcfg['weight_decay']
color_channels = rcfg['color_channels']
augmentation = rcfg['augmentation']
dataset_name = rcfg['dataset']
model_path = rcfg['model_path']
vgg_version = rcfg['vgg_version']
vgg_dropout = rcfg['vgg_dropout']
vgg_init_weights = rcfg['vgg_init_weights']
vgg_batch_norm = rcfg['vgg_batch_norm']
pred_loss_weight = rcfg['pred_loss_weight']
ae_loss_function = rcfg['ae_loss_function']
# params from global environment config
device = gcfg['device']
img_size = gcfg[f'datasets/{dataset_name}/img_size']
num_classes = gcfg[f'datasets/{dataset_name}/num_classes']
layer_configs = []
# just initialize VGG, doesnt take much time
# even when not needed
vgg = VGG(
num_classes=num_classes,
dropout=vgg_dropout,
img_size=img_size,
vgg_version=vgg_version,
batch_norm=vgg_batch_norm
)
for id_l, layer in enumerate(rcfg['layers']):
dropout_rate = layer['dropout_rate']
model_type = layer['model']
uprms = layer['upstream_params']
learning_rate = layer['learning_rate']
# Prepare the model
model = rcfg.switch(f'layers/{id_l}/model', {
'AE': lambda: SupervisedAutoencoder(
color_channels=color_channels
),
'VGGn': lambda: vgg_sidecar_layer(vgg, id_l,
dropout=dropout_rate,
kernel_size=layer['kernel_size'],
encoder_type=layer['encoder_type'],
num_classes=num_classes
),
'VGGlinear': lambda: vgg
}).to(device)
upstream = None
# Prepare the upstream for uniform autoencoder networks
if id_l < num_layers - 1 and model_type == 'AE':
upstream = rcfg.switch(f'layers/{id_l}/upstream', {
'RandomMap': lambda: RandomMap(
in_shape=uprms['in_shape'],
out_shape=uprms['out_shape']
),
'ConvMap': lambda: ConvMap(
in_shape=uprms['in_shape'],
out_shape=uprms['out_shape']
),
'DecoderMap': lambda: DecoderMap(model)
}).to(device)
# Prepare the upstream for VGG
elif model_type == 'VGGn':
_, _, _, upstream_map = vgg.get_trainable_modules()[id_l]
if upstream_map is not None:
upstream = SidecarMap([upstream_map])
# Prepare the optimizer / stack for various networks
if model_type != 'VGGlinear':
layer_type = LayerType.Stack
# Prepare learnable scalars
if pred_loss_weight == 'trainable':
tp_alpha = nn.Parameter(torch.rand(1).to(device), requires_grad=True)
trainable_params = [tp_alpha]
else:
tp_alpha = pred_loss_weight
trainable_params = []
# Init stack out of layers
prev_stack = [(cfg.model, cfg.upstream) for cfg in layer_configs]
prev_stack.append((model, upstream))
stack = NetworkStack(prev_stack).to(device)
# load stack tensors from pickle if required
stack_name = layer['pretraining_load']
if stack_name is not None:
stack_path = '{}/{}'.format(model_path, stack_name)
load_layer(stack, stack_path)
# some upstream maps require training
if upstream is not None and upstream.requires_training:
trainable_params += list(model.parameters()) + list(upstream.parameters())
else:
trainable_params += model.parameters()
elif model_type == 'VGGlinear':
stack = None
model = vgg
trainable_params = list(vgg.classifier.parameters())
layer_type = LayerType.VGGlinear
optimizer = Adam(
trainable_params,
lr=learning_rate,
weight_decay=weight_decay
)
layer_name = f'layer_{id_l}'
layer_configs.append(
LayerTrainingDefinition(
layer_type=layer_type,
layer_name=layer_name,
num_epochs=layer['num_epoch'],
upstream=upstream,
stack=stack,
model=model,
optimizer=optimizer,
pretraining_store=layer['pretraining_store'],
pretraining_load=layer['pretraining_load'],
model_base_path=model_path,
tp_alpha=tp_alpha,
ae_loss_function=ae_loss_function
)
)
# end for loop
return layer_configs
def __init__(self, gcfg: ConfigLoader, rcfg: ConfigLoader):
if gcfg['device'] == 'cuda':
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.device = gcfg['device']
self.weight_decay = rcfg['weight_decay']
self.test_every_n_epochs = rcfg['test_every_n_epochs']
self.pred_loss_weight = rcfg['pred_loss_weight']
self.waves = rcfg['waves']
self.num_classes = gcfg['datasets/{}/num_classes'.format(
rcfg['dataset'])]
color_channels = rcfg['color_channels']
data_path = gcfg['datasets/{}/path'.format(rcfg['dataset'])]
dataset_workers = gcfg['dataset_workers']
dataset_transform = rcfg['dataset_transform']
self.supervised_loader, self.unsupvised_loader,\
self.test_loader = rcfg.switch('dataset', {
'cifar10': lambda: semi_supervised_cifar10(
data_path,
dataset_transform,
supervised_ratio=rcfg['supervised_ratio'],
batch_size=rcfg['batch_size'],
augmentation=rcfg['augmentation'],
num_workers=dataset_workers
),
'cifar100': lambda: semi_supervised_cifar100(
data_path,
dataset_transform,
supervised_ratio=rcfg['supervised_ratio'],
batch_size=rcfg['batch_size'],
augmentation=rcfg['augmentation'],
num_workers=dataset_workers
),
'mnist': lambda: semi_supervised_mnist(
data_path,
supervised_ratio=rcfg['supervised_ratio'],
batch_size=rcfg['batch_size']
)
})
self.layer_configs = cfg_to_network(gcfg, rcfg)
assert len(self.supervised_loader) == len(self.unsupvised_loader)
model_detail_name = os.path.split(rcfg['model_path'])[-1]
self.writer = SummaryWriter(comment='_' + model_detail_name)
self.writer.add_text("run_config", rcfg.to_json())
self.writer.add_text("environment", gcfg.to_json())
"""
TODO: Implement in factory method to show sizes when network is built
summary(self.model, input_size=(color_channels,img_size,img_size))
"""
self.decoding_criterion = rcfg.switch(
'decoding_criterion', {
'MSELoss': lambda: nn.MSELoss(),
'BCELoss': lambda: nn.BCEWithLogitsLoss()
})
self.pred_criterion = rcfg.switch(
'prediction_criterion',
{'CrossEntropyLoss': lambda: nn.CrossEntropyLoss()})
self.test_losses = []
self.test_accs = []
def test_config_loader_from_file(self):
loader = ConfigLoader().from_file('yaml/env_template.yml')
self.assertEqual(loader['datasets/cifar10/path'], '<path>')
def test_cfg_to_network(self):
rcfg = ConfigLoader().from_file('src/yaml/nets/net_template.yml')
gcfg = ConfigLoader().from_file('src/yaml/env.yml')
net = cfg_to_network(gcfg, rcfg)
print(net)
def test_save(self):
cfg = ConfigLoader().from_file('yaml/env_template.yml')
net = AutoencoderNet(cfg['datasets/cifar10/path'])
layers: List[LayerTrainingDefinition] = net.layer_configs
net.save_layer(layers[0].model, self.file)
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('--cfg', type=file_path, required=True,
help='The main config yaml file.')
parser.add_argument('--env', type=file_path, default='src/yaml/env.yml',
help='The environment yaml file.')
args = parser.parse_args()
run_cfg_path = args.cfg
env_cfg_path = args.env
# Load configs
env_cfg = ConfigLoader().from_file(env_cfg_path)
run_cfg = ConfigLoader().from_file(run_cfg_path)
# copy the configs to dir for documentation / remembering
rcfg_fn = os.path.split(run_cfg_path)[-1]
ecfg_fn = os.path.split(env_cfg_path)[-1]
model_path = run_cfg['model_path']
if not os.path.exists(model_path):
os.makedirs(model_path)
logging.info(f'Created model path: {model_path}')
else:
logging.warning(f'Model path exists already: {model_path}')
logging.info(f'Copy {run_cfg_path} to {model_path}/{rcfg_fn}')
logging.info(f'Copy {env_cfg_path} to {model_path}/{ecfg_fn}')