def serialize_to(self, folder_path: str):
"""Creates `.crm` metadata file and `.drm` (or '.zrm' for compressed data) data file in specified path."""
# experiment_id = self.model_name + '_' + str(list(self.model_parameters.values())) + '_' + get_stamp()
file_name = self.model_name + get_stamp()
if self.use_zip:
file_extension = '.zrm'
else:
file_extension = '.drm'
drm_file_name = file_name + file_extension
crm_file_name = file_name + '.ycrm'
drm_file_path = path.join(folder_path, drm_file_name)
crm_file_path = path.join(folder_path, crm_file_name)
drm_file_path = to_safe_path(drm_file_path)
crm_file_path = to_safe_path(crm_file_path)
params_to_serialize = self.to_serializable_params(
self.model_parameters)
if self.use_zip:
save_zip(self, drm_file_path, protocol=2)
else:
with open(drm_file_path, 'wb') as drm_file:
pickle.dump(self, drm_file, protocol=2)
with open(crm_file_path, 'w') as crm_file:
yaml = YAML()
yaml.dump(
{
'model_name': self.model_name,
'model_parameters': params_to_serialize,
'data_file': drm_file_name
}, crm_file)
def load_from_data_file(drm_file_path: str):
if drm_file_path.endswith('.zrm'):
return load_zip(drm_file_path)
else:
drm_file_path = to_safe_path(drm_file_path)
with open(drm_file_path, 'rb') as drm_file:
return pickle.load(drm_file)
def _get_experiment_folder_path(self, experiment_folder):
if experiment_folder is None:
return to_safe_path(
os.path.join(get_experiment_results_folder(),
to_safe_name(self._template.experiment_name)))
return experiment_folder
def _creation_date(path_to_file):
path_to_file = to_safe_path(path_to_file)
if platform.system() == 'Windows':
return os.path.getctime(path_to_file)
else:
stat = os.stat(path_to_file)
try:
# noinspection PyUnresolvedReferences
return stat.st_birthtime
except AttributeError:
return stat.st_mtime
def browse_content_files(folder_path) -> Dict[Any, Any]:
folder_path = to_safe_path(folder_path)
data_files = {}
file_paths = list(glob.iglob(path.join(folder_path, '*.crm'))) + \
list(glob.iglob(path.join(folder_path, '*.ycrm')))
file_paths.sort(key=lambda x: SingleRunMeasurements._creation_date(x))
for file_path in file_paths:
if file_path.endswith('.ycrm'):
with open(file_path, 'r') as file:
content_dict = yaml.load(file, Loader=yaml.Loader)
else:
with open(file_path, 'rb') as file:
content_dict = pickle.load(file)
content_dict['model_parameters'] = \
SingleRunMeasurements.to_serializable_params(content_dict['model_parameters'])
ids = (content_dict['model_name'], str(content_dict['model_parameters']))
data_files[ids] = {'model_parameters': content_dict['model_parameters'],
'data_file': path.join(path.dirname(file_path), content_dict['data_file'])}
return data_files
def load_zip(filename):
"""Loads a compressed object from disk."""
filename = to_safe_path(filename)
with gzip.GzipFile(filename, 'rb') as file:
loaded_object = pickle.load(file)
return loaded_object
def _get_docs_folder_path(experiment_folder: str, experiment_name: str,
stamp: str):
return to_safe_path(
os.path.join(experiment_folder, "docs",
to_safe_name(experiment_name + stamp)))