def save_dict(self, data, path, indent=4, separators=(",", ": "), encoder_cls=MultiTypeEncoder, **kwargs): """ Saves a dict as a json file in the experiment save dir Args: data: The data to be stored as save file path: sub path in the save folder (or simply filename) indent: Indent for the json file separators: Separators for the json file encoder_cls: Encoder Class for the encoding to json """ if not path.endswith(".json"): path += ".json" path = os.path.join(self.save_dir, path) create_folder(os.path.dirname(path)) with open(path, "w") as jf: json.dump(data, jf, cls=encoder_cls, indent=indent, separators=separators, **kwargs)
def save_result(self, data, name, indent=4, separators=(",", ": "), encoder_cls=MultiTypeEncoder, **kwargs): """ Saves data as a json file in the experiment result dir Args: data: The data to be stored as result json name: name of the result json file indent: Indent for the json file separators: Separators for the json file encoder_cls: Encoder Class for the encoding to json """ if not name.endswith(".json"): name += ".json" name = os.path.join(self.result_dir, name) create_folder(os.path.dirname(name)) with open(name, "w") as jf: json.dump(data, jf, cls=encoder_cls, indent=indent, separators=separators, **kwargs)
def save_pickle(self, data, path): """ Saves a object data in the experiment save dir via pickle Args: data: The data to be stored as a save file path: sub path in the save folder (or simply filename) """ path = os.path.join(self.save_dir, path) create_folder(os.path.dirname(path)) with open(path, "wb") as out: pickle.dump(data, out)
def save_numpy_data(self, data, path): """ Saves a numpy array in the experiment save dir Args: data: The array to be stored as a save file path: sub path in the save folder (or simply filename) """ if not path.endswith(".npy"): path += ".npy" path = os.path.join(self.save_dir, path) create_folder(os.path.dirname(path)) np.save(path, data)
def __init__(self, experiment_name, base_dir, folder_format="%Y%m%d-%H%M%S_{experiment_name}", resume=False, text_logger_args=None, plot_logger_args=None, **kwargs): """ Initializes the Experiment logger and creates the experiment folder structure Args: experiment_name (str): The name of the experiment base_dir (str): The base directory in which the experiment folder will be created folder_format (str): The format for the naming of the experiment folder resume (bool): if True use the given folder and do not create new ones text_logger_args: Parameters for the TextFileLogger initialization plot_logger_args: Parameters for the NumpyPlotFileLogger initialization """ super(ExperimentLogger, self).__init__(**kwargs) self.experiment_name = experiment_name self.base_dir = base_dir self.folder_format = folder_format self.init_time = datetime.datetime.today() # try to make folder until successful or until counter runs out # this is necessary when multiple ExperimentLoggers start at the same time makedir_success = False makedir_counter = 100 makedir_exception = None while (not makedir_success and makedir_counter > 0): try: self.folder_name = self.resolve_format(folder_format, resume) self.work_dir = os.path.join(base_dir, self.folder_name) if not resume: create_folder(self.work_dir) makedir_success = True except FileExistsError as file_error: makedir_counter -= 1 except Exception as e: makedir_exception = e makedir_counter -= 1 if makedir_exception is not None: warnings.warn("Last exception encountered in makedir process:\n" + "{}\n".format(makedir_exception) + "There may or may not be a folder for the experiment to run in.", RuntimeWarning) self.config_dir = os.path.join(self.work_dir, "config") self.log_dir = os.path.join(self.work_dir, "log") self.checkpoint_dir = os.path.join(self.work_dir, "checkpoint") self.img_dir = os.path.join(self.work_dir, "img") self.plot_dir = os.path.join(self.work_dir, "plot") self.save_dir = os.path.join(self.work_dir, "save") self.result_dir = os.path.join(self.work_dir, "result") if not resume: create_folder(self.config_dir) create_folder(self.log_dir) create_folder(self.checkpoint_dir) create_folder(self.img_dir) create_folder(self.plot_dir) create_folder(self.save_dir) create_folder(self.result_dir) if text_logger_args is None: text_logger_args = {} if plot_logger_args is None: plot_logger_args = {} self.text_logger = TextFileLogger(self.log_dir, **text_logger_args) self.plot_logger = NumpyPlotFileLogger( self.img_dir, self.plot_dir, **plot_logger_args)