def setup_training(args: argparse.Namespace) -> None:
"""
Sets up directories (logging and tensorboard), sets up the general logger and calls
train_model(...) with the parameters specified in args for this run.
The model to train can be a complete model with all the tiers (if args.tier == None) or a
single tier (the one specified in args.tier).
Args:
args (argparse.Namespace): parameters to set up the training. At least, args must contain:
args = {"path_config": ...,
"tier": ...,
"checkpoint_path": ...}
"""
# 1. Read hyperparameters from file
hp = HParams.from_yaml(args.path_config)
# check if GPU available and add it to parameters
hp["device"] = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
# 2. Create extension of the architecture of the model and timestamp for this run (use to
# identify folders and files created for this run)
# format: f(params_file)_t(n_tiers)_l(n_layers)_hd(hidden_size)_gmm(gmm_size).
extension_architecture = f"d{hp.name}_t{hp.network.n_tiers}_" \
f"l{'.'.join(map(str, hp.network.layers))}_" \
f"hd{hp.network.hidden_size}_gmm{hp.network.gmm_size}"
timestamp = f"{datetime.now().strftime('%Y%m%d-%H%M%S')}"
# 3 Create directories for saving logs and model weights if they do not exist
# 3.1 Create model weights directory for this run (the same directory will be used for different
# runs of a model with same architecture and the difference will be in the file stored)
hp["training"][
"dir_chkpt"] = hp.training.dir_chkpt + extension_architecture
Path(hp.training.dir_chkpt).mkdir(parents=True, exist_ok=True)
# 3.2 Create general log directory for this run (the same directory will be used for different
# runs of a model with same architecture and the difference will be in the file stored)
hp["logging"]["dir_log"] = hp.logging.dir_log + extension_architecture
Path(hp.logging.dir_log).mkdir(parents=True, exist_ok=True)
# 4. Setup general logging (it will use the folder previously created and the filename will be:
tier = str(args.tier) if args.tier is not None else 'ALL'
filename = f"{hp.logging.dir_log}/tier{tier}_{timestamp}"
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
filename=filename), # handler to save the log to a file
logging.StreamHandler(
) # handler to output the log to the terminal
])
logger = logging.getLogger()
# 5. Show device that will be used for training: CPU or GPU
logger.info(f"Device for training: {hp.device}")
# 6. Start training of the model (or a single tier, depending on args)
train_model(args, hp, extension_architecture, timestamp, logger)
def setup_synthesize(args: argparse.Namespace):
"""
Sets up synthesis with the parameters specified in args and the path to the weights of the model
Args:
args (argparse.Namespace): parameters to set up the synthesis.
"""
# 1. Read hyperparameters from file
hp = HParams.from_yaml(args.path_config)
synthesisp = HParams.from_yaml(args.path_synthesis)
# check if GPU available and add it to parameters
hp["device"] = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
# 2. Create extension of the architecture of the model and timestamp for this run (use to
# identify folders and files created for this run)
# format: f(params_file)_t(n_tiers)_l(n_layers)_hd(hidden_size)_gmm(gmm_size).
extension_architecture = f"d{hp.name}_t{hp.network.n_tiers}_" \
f"l{'.'.join(map(str, hp.network.layers))}_" \
f"hd{hp.network.hidden_size}_gmm{hp.network.gmm_size}"
timestamp = f"{datetime.now().strftime('%Y%m%d-%H%M%S')}"
# 3 Create directories for saving logs and output if they do not exist
# 3.1 Create general log directory for this run (the same directory will be used for different
# runs of a model with same architecture and the difference will be in the file stored)
hp["logging"]["dir_log"] = hp.logging.dir_log + extension_architecture
Path(hp.logging.dir_log).mkdir(parents=True, exist_ok=True)
# 3.2 Create directory for the outputs of this run (the same directory will be used for
# different runs of a model with same architecture and the difference will be in the weights
# of the model)
synthesisp.output_path = synthesisp.output_path + extension_architecture
Path(synthesisp.output_path).mkdir(parents=True, exist_ok=True)
# 4. Setup general logging (it will use the folder previously created and the filename will be:
filename = f"{hp.logging.dir_log}/synthesis_{timestamp}"
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
filename=filename), # handler to save the log to a file
logging.StreamHandler(
) # handler to output the log to the terminal
])
logger = logging.getLogger()
# 5. Show device that will be used for training: CPU or GPU
logger.info(f"Device for training: {hp.device}")
# 6. Setup tensorboard logging
# 6.1 Create tensorboard logs directory (tensorboard requires a different folder for
# each run of the model, in this case every run to train a tier) so we add the extension
# of the network's architecture of this run and the timestamp to identify it completely
tensorboard_dir = hp.logging.dir_log_tensorboard + extension_architecture \
+ f"synthesis_{timestamp}"
Path(tensorboard_dir).mkdir(parents=True, exist_ok=True)
# 2.2 Create tensorboard writer
tensorboardwriter = TensorboardWriter(hp, tensorboard_dir)
synthesize(args, hp, synthesisp, extension_architecture, timestamp,
tensorboardwriter, logger)
tensorboardwriter.close()