def save_dict(self,
data,
path,
indent=4,
separators=(",", ": "),
encoder_cls=MultiTypeEncoder,
**kwargs):
"""
Saves a dict as a json file in the experiment save dir
Args:
data: The data to be stored as save file
path: sub path in the save folder (or simply filename)
indent: Indent for the json file
separators: Separators for the json file
encoder_cls: Encoder Class for the encoding to json
"""
if not path.endswith(".json"):
path += ".json"
path = os.path.join(self.save_dir, path)
create_folder(os.path.dirname(path))
with open(path, "w") as jf:
json.dump(data,
jf,
cls=encoder_cls,
indent=indent,
separators=separators,
**kwargs)
def save_result(self,
data,
name,
indent=4,
separators=(",", ": "),
encoder_cls=MultiTypeEncoder,
**kwargs):
"""
Saves data as a json file in the experiment result dir
Args:
data: The data to be stored as result json
name: name of the result json file
indent: Indent for the json file
separators: Separators for the json file
encoder_cls: Encoder Class for the encoding to json
"""
if not name.endswith(".json"):
name += ".json"
name = os.path.join(self.result_dir, name)
create_folder(os.path.dirname(name))
with open(name, "w") as jf:
json.dump(data,
jf,
cls=encoder_cls,
indent=indent,
separators=separators,
**kwargs)
def save_pickle(self, data, path):
"""
Saves a object data in the experiment save dir via pickle
Args:
data: The data to be stored as a save file
path: sub path in the save folder (or simply filename)
"""
path = os.path.join(self.save_dir, path)
create_folder(os.path.dirname(path))
with open(path, "wb") as out:
pickle.dump(data, out)
def save_numpy_data(self, data, path):
"""
Saves a numpy array in the experiment save dir
Args:
data: The array to be stored as a save file
path: sub path in the save folder (or simply filename)
"""
if not path.endswith(".npy"):
path += ".npy"
path = os.path.join(self.save_dir, path)
create_folder(os.path.dirname(path))
np.save(path, data)
def __init__(self,
experiment_name,
base_dir,
folder_format="%Y%m%d-%H%M%S_{experiment_name}",
resume=False,
text_logger_args=None,
plot_logger_args=None,
**kwargs):
"""
Initializes the Experiment logger and creates the experiment folder structure
Args:
experiment_name (str): The name of the experiment
base_dir (str): The base directory in which the experiment folder will be created
folder_format (str): The format for the naming of the experiment folder
resume (bool): if True use the given folder and do not create new ones
text_logger_args: Parameters for the TextFileLogger initialization
plot_logger_args: Parameters for the NumpyPlotFileLogger initialization
"""
super(ExperimentLogger, self).__init__(**kwargs)
self.experiment_name = experiment_name
self.base_dir = base_dir
self.folder_format = folder_format
self.init_time = datetime.datetime.today()
# try to make folder until successful or until counter runs out
# this is necessary when multiple ExperimentLoggers start at the same time
makedir_success = False
makedir_counter = 100
makedir_exception = None
while (not makedir_success and makedir_counter > 0):
try:
self.folder_name = self.resolve_format(folder_format, resume)
self.work_dir = os.path.join(base_dir, self.folder_name)
if not resume:
create_folder(self.work_dir)
makedir_success = True
except FileExistsError as file_error:
makedir_counter -= 1
except Exception as e:
makedir_exception = e
makedir_counter -= 1
if makedir_exception is not None:
warnings.warn("Last exception encountered in makedir process:\n" +
"{}\n".format(makedir_exception) +
"There may or may not be a folder for the experiment to run in.", RuntimeWarning)
self.config_dir = os.path.join(self.work_dir, "config")
self.log_dir = os.path.join(self.work_dir, "log")
self.checkpoint_dir = os.path.join(self.work_dir, "checkpoint")
self.img_dir = os.path.join(self.work_dir, "img")
self.plot_dir = os.path.join(self.work_dir, "plot")
self.save_dir = os.path.join(self.work_dir, "save")
self.result_dir = os.path.join(self.work_dir, "result")
if not resume:
create_folder(self.config_dir)
create_folder(self.log_dir)
create_folder(self.checkpoint_dir)
create_folder(self.img_dir)
create_folder(self.plot_dir)
create_folder(self.save_dir)
create_folder(self.result_dir)
if text_logger_args is None:
text_logger_args = {}
if plot_logger_args is None:
plot_logger_args = {}
self.text_logger = TextFileLogger(self.log_dir, **text_logger_args)
self.plot_logger = NumpyPlotFileLogger(
self.img_dir, self.plot_dir, **plot_logger_args)
