def serialize_to(self, folder_path: str): """Creates `.crm` metadata file and `.drm` (or '.zrm' for compressed data) data file in specified path.""" # experiment_id = self.model_name + '_' + str(list(self.model_parameters.values())) + '_' + get_stamp() file_name = self.model_name + get_stamp() if self.use_zip: file_extension = '.zrm' else: file_extension = '.drm' drm_file_name = file_name + file_extension crm_file_name = file_name + '.ycrm' drm_file_path = path.join(folder_path, drm_file_name) crm_file_path = path.join(folder_path, crm_file_name) drm_file_path = to_safe_path(drm_file_path) crm_file_path = to_safe_path(crm_file_path) params_to_serialize = self.to_serializable_params( self.model_parameters) if self.use_zip: save_zip(self, drm_file_path, protocol=2) else: with open(drm_file_path, 'wb') as drm_file: pickle.dump(self, drm_file, protocol=2) with open(crm_file_path, 'w') as crm_file: yaml = YAML() yaml.dump( { 'model_name': self.model_name, 'model_parameters': params_to_serialize, 'data_file': drm_file_name }, crm_file)
def load_from_data_file(drm_file_path: str): if drm_file_path.endswith('.zrm'): return load_zip(drm_file_path) else: drm_file_path = to_safe_path(drm_file_path) with open(drm_file_path, 'rb') as drm_file: return pickle.load(drm_file)
def _get_experiment_folder_path(self, experiment_folder): if experiment_folder is None: return to_safe_path( os.path.join(get_experiment_results_folder(), to_safe_name(self._template.experiment_name))) return experiment_folder
def _creation_date(path_to_file): path_to_file = to_safe_path(path_to_file) if platform.system() == 'Windows': return os.path.getctime(path_to_file) else: stat = os.stat(path_to_file) try: # noinspection PyUnresolvedReferences return stat.st_birthtime except AttributeError: return stat.st_mtime
def browse_content_files(folder_path) -> Dict[Any, Any]: folder_path = to_safe_path(folder_path) data_files = {} file_paths = list(glob.iglob(path.join(folder_path, '*.crm'))) + \ list(glob.iglob(path.join(folder_path, '*.ycrm'))) file_paths.sort(key=lambda x: SingleRunMeasurements._creation_date(x)) for file_path in file_paths: if file_path.endswith('.ycrm'): with open(file_path, 'r') as file: content_dict = yaml.load(file, Loader=yaml.Loader) else: with open(file_path, 'rb') as file: content_dict = pickle.load(file) content_dict['model_parameters'] = \ SingleRunMeasurements.to_serializable_params(content_dict['model_parameters']) ids = (content_dict['model_name'], str(content_dict['model_parameters'])) data_files[ids] = {'model_parameters': content_dict['model_parameters'], 'data_file': path.join(path.dirname(file_path), content_dict['data_file'])} return data_files
def load_zip(filename): """Loads a compressed object from disk.""" filename = to_safe_path(filename) with gzip.GzipFile(filename, 'rb') as file: loaded_object = pickle.load(file) return loaded_object
def _get_docs_folder_path(experiment_folder: str, experiment_name: str, stamp: str): return to_safe_path( os.path.join(experiment_folder, "docs", to_safe_name(experiment_name + stamp)))