def patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]: patch = load_yml(patch_path) experiment_base = patch['experiment_name'] out_folder = patch['yml_output_folder'] mkdir(out_folder) for key, values in patch['run_cfg'].items(): for value in values: cfg = ConfigLoader().from_file(cfg_path, suppress_print=True) broadcast = key.split('/') broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1: layer_key = key.split('/')[-1] for i in [int(j) for j in broadcast]: b_key = f'layers/{i}/{layer_key}' cfg[b_key] = value assert cfg[b_key] == value else: cfg[key] = value assert cfg[key] == value param_name = key.replace('/','_').split('_') param_name = ''.join([s[0] for s in param_name]) str_value = str(value).replace('.', '_') ex_uuid = uuid.uuid4().hex[0:6] experiment_name = f'{experiment_base}_{ex_uuid}_{param_name}={str_value}' cfg['model_path'] = f'trained_models/{experiment_name}' out_path = f'{out_folder}/{experiment_name}.yml' logging.info(f'Saving param: {key} value: {value} to {out_path}') save_yml(cfg.env, out_path)
def test_array(self): document = """ l: - list: str attr: 1 - test: nr attr: 2 """ loader = ConfigLoader().from_string(document) self.assertEqual(loader['l/0/attr'], 1) self.assertEqual(loader['l/1/attr'], 2) loader['l/0/attr'] = 3 loader['l/1/attr'] = 4 self.assertEqual(loader['l/0/attr'], 3) self.assertEqual(loader['l/1/attr'], 4)
def test_config_loader_from_str(self): document = """ a: 1 b: c: 3 d: 4 """ loader = ConfigLoader().from_string(document) self.assertEqual(loader['b/c'], 3) self.assertEqual(loader['a'], 1) loader['a'] = 'bär' self.assertEqual(loader['a'], 'bär') self.assertEqual(loader['b']['c'], 3) self.assertEqual(loader['b']['d'], 4) self.assertEqual(loader['b', 'c'], 3)
def permute_patch_to_cfgs(cfg_path:str, patch_path: str) -> List[ConfigLoader]: patch = load_yml(patch_path) experiment_base = patch['experiment_name'] out_folder = patch['yml_output_folder'] mkdir(out_folder) keys = [] values = [] for key, vals in patch['run_cfg'].items(): keys += [key] values.append(vals) permutations = list(product(*values)) num_permutations = len(permutations) logging.info(f"Generating {num_permutations} Permuted Configurations") # iterate over all permutation tuples for tple in permutations: fn_prefix = '' cfg = ConfigLoader().from_file(cfg_path, suppress_print=True) for id_k, key in enumerate(keys): broadcast = key.split('/') broadcast = broadcast[1].split(',') if len(broadcast) > 1 else None if key.startswith('layers/') and broadcast is not None and len(broadcast) > 1: layer_key = key.split('/')[-1] for i in [int(j) for j in broadcast]: b_key = f'layers/{i}/{layer_key}' cfg[b_key] = tple[id_k] assert cfg[b_key] == tple[id_k] else: cfg[key] = tple[id_k] assert cfg[key] == tple[id_k] param_name = key.replace('/','_').split('_') param_name = ''.join([s[0] for s in param_name]) str_value = str(tple[id_k]).replace('.', '_') fn_prefix += f'_{param_name}={str_value}' ex_uuid = uuid.uuid4().hex[0:6] experiment_name = f'{experiment_base}_{ex_uuid}_{fn_prefix}' cfg['model_path'] = f'trained_models/{experiment_name}' out_path = f'{out_folder}/{experiment_name}.yml' logging.info(f'Saving tuple: {tple} to {out_path}') save_yml(cfg.env, out_path)
def cfg_to_network(gcfg: ConfigLoader, rcfg: ConfigLoader) \ -> List[LayerTrainingDefinition]: r""" This is the main network assembly function. It takes a config from a configloader (a yaml file). Assembles a network stack as list of LayerTrainingDefintions. Returns: List[LayerTrainingDefinition] """ # params from run config num_layers = len(rcfg['layers']) weight_decay = rcfg['weight_decay'] color_channels = rcfg['color_channels'] augmentation = rcfg['augmentation'] dataset_name = rcfg['dataset'] model_path = rcfg['model_path'] vgg_version = rcfg['vgg_version'] vgg_dropout = rcfg['vgg_dropout'] vgg_init_weights = rcfg['vgg_init_weights'] vgg_batch_norm = rcfg['vgg_batch_norm'] pred_loss_weight = rcfg['pred_loss_weight'] ae_loss_function = rcfg['ae_loss_function'] # params from global environment config device = gcfg['device'] img_size = gcfg[f'datasets/{dataset_name}/img_size'] num_classes = gcfg[f'datasets/{dataset_name}/num_classes'] layer_configs = [] # just initialize VGG, doesnt take much time # even when not needed vgg = VGG( num_classes=num_classes, dropout=vgg_dropout, img_size=img_size, vgg_version=vgg_version, batch_norm=vgg_batch_norm ) for id_l, layer in enumerate(rcfg['layers']): dropout_rate = layer['dropout_rate'] model_type = layer['model'] uprms = layer['upstream_params'] learning_rate = layer['learning_rate'] # Prepare the model model = rcfg.switch(f'layers/{id_l}/model', { 'AE': lambda: SupervisedAutoencoder( color_channels=color_channels ), 'VGGn': lambda: vgg_sidecar_layer(vgg, id_l, dropout=dropout_rate, kernel_size=layer['kernel_size'], encoder_type=layer['encoder_type'], num_classes=num_classes ), 'VGGlinear': lambda: vgg }).to(device) upstream = None # Prepare the upstream for uniform autoencoder networks if id_l < num_layers - 1 and model_type == 'AE': upstream = rcfg.switch(f'layers/{id_l}/upstream', { 'RandomMap': lambda: RandomMap( in_shape=uprms['in_shape'], out_shape=uprms['out_shape'] ), 'ConvMap': lambda: ConvMap( in_shape=uprms['in_shape'], out_shape=uprms['out_shape'] ), 'DecoderMap': lambda: DecoderMap(model) }).to(device) # Prepare the upstream for VGG elif model_type == 'VGGn': _, _, _, upstream_map = vgg.get_trainable_modules()[id_l] if upstream_map is not None: upstream = SidecarMap([upstream_map]) # Prepare the optimizer / stack for various networks if model_type != 'VGGlinear': layer_type = LayerType.Stack # Prepare learnable scalars if pred_loss_weight == 'trainable': tp_alpha = nn.Parameter(torch.rand(1).to(device), requires_grad=True) trainable_params = [tp_alpha] else: tp_alpha = pred_loss_weight trainable_params = [] # Init stack out of layers prev_stack = [(cfg.model, cfg.upstream) for cfg in layer_configs] prev_stack.append((model, upstream)) stack = NetworkStack(prev_stack).to(device) # load stack tensors from pickle if required stack_name = layer['pretraining_load'] if stack_name is not None: stack_path = '{}/{}'.format(model_path, stack_name) load_layer(stack, stack_path) # some upstream maps require training if upstream is not None and upstream.requires_training: trainable_params += list(model.parameters()) + list(upstream.parameters()) else: trainable_params += model.parameters() elif model_type == 'VGGlinear': stack = None model = vgg trainable_params = list(vgg.classifier.parameters()) layer_type = LayerType.VGGlinear optimizer = Adam( trainable_params, lr=learning_rate, weight_decay=weight_decay ) layer_name = f'layer_{id_l}' layer_configs.append( LayerTrainingDefinition( layer_type=layer_type, layer_name=layer_name, num_epochs=layer['num_epoch'], upstream=upstream, stack=stack, model=model, optimizer=optimizer, pretraining_store=layer['pretraining_store'], pretraining_load=layer['pretraining_load'], model_base_path=model_path, tp_alpha=tp_alpha, ae_loss_function=ae_loss_function ) ) # end for loop return layer_configs
def __init__(self, gcfg: ConfigLoader, rcfg: ConfigLoader): if gcfg['device'] == 'cuda': self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.device = gcfg['device'] self.weight_decay = rcfg['weight_decay'] self.test_every_n_epochs = rcfg['test_every_n_epochs'] self.pred_loss_weight = rcfg['pred_loss_weight'] self.waves = rcfg['waves'] self.num_classes = gcfg['datasets/{}/num_classes'.format( rcfg['dataset'])] color_channels = rcfg['color_channels'] data_path = gcfg['datasets/{}/path'.format(rcfg['dataset'])] dataset_workers = gcfg['dataset_workers'] dataset_transform = rcfg['dataset_transform'] self.supervised_loader, self.unsupvised_loader,\ self.test_loader = rcfg.switch('dataset', { 'cifar10': lambda: semi_supervised_cifar10( data_path, dataset_transform, supervised_ratio=rcfg['supervised_ratio'], batch_size=rcfg['batch_size'], augmentation=rcfg['augmentation'], num_workers=dataset_workers ), 'cifar100': lambda: semi_supervised_cifar100( data_path, dataset_transform, supervised_ratio=rcfg['supervised_ratio'], batch_size=rcfg['batch_size'], augmentation=rcfg['augmentation'], num_workers=dataset_workers ), 'mnist': lambda: semi_supervised_mnist( data_path, supervised_ratio=rcfg['supervised_ratio'], batch_size=rcfg['batch_size'] ) }) self.layer_configs = cfg_to_network(gcfg, rcfg) assert len(self.supervised_loader) == len(self.unsupvised_loader) model_detail_name = os.path.split(rcfg['model_path'])[-1] self.writer = SummaryWriter(comment='_' + model_detail_name) self.writer.add_text("run_config", rcfg.to_json()) self.writer.add_text("environment", gcfg.to_json()) """ TODO: Implement in factory method to show sizes when network is built summary(self.model, input_size=(color_channels,img_size,img_size)) """ self.decoding_criterion = rcfg.switch( 'decoding_criterion', { 'MSELoss': lambda: nn.MSELoss(), 'BCELoss': lambda: nn.BCEWithLogitsLoss() }) self.pred_criterion = rcfg.switch( 'prediction_criterion', {'CrossEntropyLoss': lambda: nn.CrossEntropyLoss()}) self.test_losses = [] self.test_accs = []
def test_config_loader_from_file(self): loader = ConfigLoader().from_file('yaml/env_template.yml') self.assertEqual(loader['datasets/cifar10/path'], '<path>')
def test_cfg_to_network(self): rcfg = ConfigLoader().from_file('src/yaml/nets/net_template.yml') gcfg = ConfigLoader().from_file('src/yaml/env.yml') net = cfg_to_network(gcfg, rcfg) print(net)
def test_save(self): cfg = ConfigLoader().from_file('yaml/env_template.yml') net = AutoencoderNet(cfg['datasets/cifar10/path']) layers: List[LayerTrainingDefinition] = net.layer_configs net.save_layer(layers[0].model, self.file)
# Parse arguments parser = argparse.ArgumentParser() parser.add_argument('--cfg', type=file_path, required=True, help='The main config yaml file.') parser.add_argument('--env', type=file_path, default='src/yaml/env.yml', help='The environment yaml file.') args = parser.parse_args() run_cfg_path = args.cfg env_cfg_path = args.env # Load configs env_cfg = ConfigLoader().from_file(env_cfg_path) run_cfg = ConfigLoader().from_file(run_cfg_path) # copy the configs to dir for documentation / remembering rcfg_fn = os.path.split(run_cfg_path)[-1] ecfg_fn = os.path.split(env_cfg_path)[-1] model_path = run_cfg['model_path'] if not os.path.exists(model_path): os.makedirs(model_path) logging.info(f'Created model path: {model_path}') else: logging.warning(f'Model path exists already: {model_path}') logging.info(f'Copy {run_cfg_path} to {model_path}/{rcfg_fn}') logging.info(f'Copy {env_cfg_path} to {model_path}/{ecfg_fn}')