def _post_process_model_config(model_config, config, should_make_dir):
"""Method to post process the model section of the config"""
general_config = config.general
if not model_config.save_dir:
model_config.save_dir = os.path.join(general_config.base_path,
"models", general_config.id)
elif model_config.save_dir[0] != "/":
model_config.save_dir = os.path.join(general_config.base_path,
model_config.save_dir)
if should_make_dir:
make_dir(path=model_config.save_dir)
# model_config.load_path = os.path.join(general_config.base_path,
# "model", model_config.load_path)
if not model_config.load_dir:
model_config.load_dir = os.path.join(general_config.base_path,
"models")
elif model_config.save_dir[0] != "/":
model_config.load_dir = os.path.join(general_config.base_path,
"models", model_config.load_dir)
for key in ["learning_rate", "eps"]:
model_config.optim[key] = float(model_config.optim[key])
return model_config
def _post_process_expert_policy_config(expert_policy_config, general_config,
should_make_dir):
# Method to post process the expert_policy section of the config
default_params = {
"name": "ppo",
"num_timesteps": 1000000,
"save_dir": "",
"hidden_size": 64,
"num_layers": 2,
"num_cpu": 1,
"should_convert_model": True
}
expert_policy_config = _copy_from_default_dict(expert_policy_config,
default_params)
if (expert_policy_config.save_dir == ""):
expert_policy_config.save_dir = os.path.join(general_config.base_path,
"model", "expert_policy")
elif (expert_policy_config.save_dir[0] != "/"):
expert_policy_config.save_dir = os.path.join(general_config.base_path,
"model", "expert_policy")
if (should_make_dir):
make_dir(path=expert_policy_config.save_dir)
return expert_policy_config
def _post_process_plot_config(plot_config, general_config):
# Method to post process the plot section of the config
if ("base_path" not in plot_config) or (plot_config.base_path == ""):
plot_config.base_path = os.path.join(general_config.base_path,
"plots", general_config.id)
make_dir(path=plot_config.base_path)
return plot_config
def _post_process_plot_config(plot_config, general_config, should_make_dir):
"""Method to post process the plot section of the config"""
if not plot_config.base_path:
plot_config.base_path = os.path.join(general_config.base_path, "plots",
general_config.id)
if should_make_dir:
make_dir(path=plot_config.base_path)
return plot_config
def _post_process_log_config(log_config, general_config, should_make_dir):
# Method to post process the log section of the config
if ("file_path" not in log_config) or (log_config.file_path == ""):
log_config.file_path = os.path.join(general_config.base_path, "logs",
general_config.id)
if (should_make_dir):
make_dir(path=log_config.file_path)
log_config.file_path = os.path.join(log_config.file_path, "log.txt")
log_config.dir = log_config.file_path.rsplit("/", 1)[0]
return log_config
def _post_process_model_config(model_config, config):
# Method to post process the net section of the config
general_config = config.general
dataset_config = config.dataset
default_params = {
"name": "baseline1",
"batch_size": 100,
"num_epochs": 1000,
"persist_per_epoch": -1,
"early_stopping_patience": 1,
"save_dir": "",
"should_load_model": False,
"dropout_probability": 0.0,
"embedding": {},
"optimiser": {},
"early_stopping": {}
}
for key in default_params:
if key not in model_config:
model_config[key] = default_params[key]
if (model_config.save_dir == ""):
model_config.save_dir = os.path.join(general_config.base_path, "net",
general_config.id)
elif (model_config.save_dir[0] != "/"):
model_config.save_dir = os.path.join(general_config.base_path,
model_config.save_dir)
make_dir(path=model_config.save_dir)
model_config.load_path = os.path.join(general_config.base_path, "net",
general_config.id)
for key in ["should_load_model"]:
model_config[key] = _get_boolean_value(model_config[key])
model_config.early_stopping = _post_process_early_stooping_config(
model_config.early_stopping)
model_config.embedding = _post_process_embedding_config(
deepcopy(model_config.embedding), general_config, dataset_config)
model_config.optimiser = _post_process_optimiser_config(
deepcopy(model_config.optimiser))
model_config.decoder = _post_process_decoder_config(
deepcopy(model_config.decoder), model_config.encoder)
return model_config
def _post_process_model_config(model_config, config, should_make_dir):
# Method to post process the model section of the config
general_config = config.general
default_params = {
"name": "baseline1",
"should_train": False,
"num_epochs": 1000,
"persist_per_epoch": -1,
"persist_best_model": False,
"early_stopping_patience": 1,
"save_dir": "",
"should_load_model": False,
"optimizer": Dict(),
"imagination_model": Dict(),
"expert_policy": Dict(),
"modes": ["train", "val", "test"],
}
for key in default_params:
if key not in model_config:
model_config[key] = default_params[key]
if (model_config.save_dir == ""):
model_config.save_dir = os.path.join(general_config.base_path, "model",
general_config.id)
elif (model_config.save_dir[0] != "/"):
model_config.save_dir = os.path.join(general_config.base_path,
model_config.save_dir)
if (should_make_dir):
make_dir(path=model_config.save_dir)
model_config.load_path = os.path.join(general_config.base_path, "model",
model_config.load_path)
for key in ["should_load_model"]:
model_config[key] = _get_boolean_value(model_config[key])
model_config.optimizer = _post_process_optimizer_config(
deepcopy(model_config.optimizer))
model_config.imagination_model = _post_process_imagination_model_config(
deepcopy(model_config.imagination_model))
model_config.expert_policy = _post_process_expert_policy_config(
deepcopy(model_config.expert_policy), general_config, should_make_dir)
return model_config
def make_remote_config(sample_config, app_id):
remote_config = deepcopy(sample_config)
remote_config.general.id = app_id
# Log Params
key = "file_path"
remote_config.log[key] = os.path.join(remote_config.general.base_path,
"logs", remote_config.general.id)
remote_config.log[key] = os.path.join(remote_config.log[key], "log.txt")
# Plot Params
remote_config.plot.base_path = os.path.join(
remote_config.general.base_path, "plots", remote_config.general.id)
make_dir(remote_config.plot.base_path)
return remote_config
def save(
self,
save_dir,
epoch: Optional[int] = None,
composition_fn: Optional[nn.Module] = None,
representation_fn: Optional[nn.Module] = None,
optimizers: Iterable[torch.optim.Optimizer] = None,
) -> None:
"""Persist the model weights in disk
Arguments:
save_dir {str} -- save location
Keyword Arguments:
epoch {Optional[int]} -- [description] (default: {None})
composition_fn {Optional[nn.Module]} -- Graph Function (default: {None})
representation_fn {Optional[nn.Module]} -- previously signature function (default: {None})
optimizers {Iterable[torch.optim.Optimizer]} -- [description] (default: {None})
"""
if epoch is None:
epoch = 0
path_to_save_model_at = os.path.join(
save_dir,
str(epoch),
)
make_dir(path_to_save_model_at)
data = {
"composition_fn": {
"weights": composition_fn.weights,
"weight_names": composition_fn.weight_names,
},
"representation_fn": {
"weights": representation_fn.weights,
"weight_names": representation_fn.weight_names,
},
"optimizers": [opt.state_dict() for opt in optimizers],
}
torch.save(data, os.path.join(path_to_save_model_at, "model.pt"))
data = {"epoch": epoch}
path_to_save_metadata_at = os.path.join(save_dir, )
make_dir(path_to_save_metadata_at)
torch.save(data, os.path.join(path_to_save_metadata_at, "metadata.pt"))
def _post_process_dataset_config(dataset_config, general_config,
should_make_dir):
# Method to post process the dataset section of the config
if ("base_path" not in dataset_config) or (dataset_config.base_path == ""):
dataset_config.base_path = os.path.join(general_config.base_path,
"data")
if (should_make_dir):
make_dir(dataset_config.base_path)
if ("name" not in dataset_config) or (dataset_config.name == ""):
dataset_config.name = "HalfCheetah-v2"
if (should_make_dir):
make_dir(os.path.join(dataset_config.base_path, dataset_config.name))
default_params = {
"should_generate": False,
"num_trajectories": 10000,
"num_actions_per_trajectory": 1,
"batch_size": 100,
"buffer_size": 100,
"num_workers": 2,
"should_load": True,
"sequence_length": 250,
"imagination_length": 10,
}
dataset_config = _copy_from_default_dict(dataset_config, default_params)
for key in ["should_generate"]:
if key in dataset_config:
dataset_config[key] = _get_boolean_value(dataset_config[key])
dataset_config.split = _post_process_split_config(
deepcopy(dataset_config.split))
dataset_config.dataset_generation = _post_process_dataset_generation_config(
deepcopy(dataset_config.dataset_generation))
return dataset_config
def _post_process_log_config(log_config, general_config):
# Method to post process the log section of the config
if ("file_path" not in log_config) or (log_config.file_path == ""):
log_config.file_path = os.path.join(general_config.base_path,
"logs", general_config.id)
make_dir(path=log_config.file_path)
log_config.file_path = os.path.join(log_config.file_path, "log.txt")
log_config.dir = log_config.file_path.rsplit("/", 1)[0]
key = "mongo_host"
if (key not in log_config or log_config[key] == ""):
log_config[key] = "127.0.0.1"
key = "mongo_port"
if (key not in log_config or log_config[key] == ""):
log_config[key] = "8092"
key = "mongo_db"
if (key not in log_config or log_config[key] == ""):
log_config[key] = "graphsum"
return log_config
def _post_process_logger_file_config(logger_file_config, general_config,
should_make_dir):
"""Method to post process the file subsection of the logger section of the config"""
if not logger_file_config.path:
logger_file_config.path = os.path.join(general_config.base_path,
"logs", general_config.id)
if should_make_dir:
make_dir(path=logger_file_config.path)
make_dir(os.path.join(logger_file_config.path, "train"))
make_dir(os.path.join(logger_file_config.path, "eval"))
logger_file_config.path = os.path.join(logger_file_config.path,
"log.txt")
logger_file_config.dir = logger_file_config.path.rsplit("/", 1)[0]
return logger_file_config
def get_config_from_log(log):
"""Method to prepare the config for all downstream tasks"""
boxed_config = get_config_box(log)
boxed_config.general.base_path = os.path.dirname(
os.path.realpath(__file__)).split("/codes")[0]
boxed_config.logger.file.path = os.path.join(
boxed_config.general.base_path, "logs", boxed_config.general.id)
make_dir(path=boxed_config.logger.file.path)
make_dir(os.path.join(boxed_config.logger.file.path, "train"))
make_dir(os.path.join(boxed_config.logger.file.path, "eval"))
boxed_config.logger.file.path = os.path.join(boxed_config.logger.file.path,
"log.txt")
boxed_config.logger.file.dir = boxed_config.logger.file.path.rsplit(
"/", 1)[0]
return boxed_config
