def __init__(self,
logging_directory: str,
custom_logger: Optional[Logger] = None,
with_mlflow: bool = False,
if_exists: str = 'error'):
logging_directory = _check_directory(logging_directory, if_exists)
os.makedirs(logging_directory, exist_ok=True)
self.logging_directory = logging_directory
self.with_mlflow = with_mlflow
if custom_logger is not None:
self.logger = custom_logger
self.is_custom = True
else:
self.logger = getLogger(str(uuid.uuid4()))
self.log_path = os.path.join(logging_directory, 'log.txt')
self.logger.addHandler(FileHandler(self.log_path))
self.logger.setLevel(DEBUG)
self.is_custom = False
self.metrics = self._load_dict('metrics.json')
self.params = self._load_dict('params.json')
self.inherit_existing_run = False
if self.with_mlflow:
requires_mlflow()
self.mlflow_run_id = _try_to_get_existing_mlflow_run_id(
logging_directory)
if self.mlflow_run_id is not None:
self.mlflow_run_name = None
else:
self.mlflow_run_name = logging_directory
def _check_directory(directory: str, if_exists: str) -> str:
if os.path.exists(directory):
if if_exists == 'error':
raise ValueError('directory {} already exists.'.format(directory))
elif if_exists == 'replace':
warnings.warn(
'directory {} already exists. It will be replaced by the new result'
.format(directory))
existing_run_id = _try_to_get_existing_mlflow_run_id(directory)
if existing_run_id is not None:
requires_mlflow()
import mlflow
mlflow.delete_run(existing_run_id)
shutil.rmtree(directory, ignore_errors=True)
elif if_exists == 'rename':
postfix_index = 1
while os.path.exists(directory + '_' + str(postfix_index)):
postfix_index += 1
directory += '_' + str(postfix_index)
warnings.warn(
'directory is renamed to {} because the original directory already exists.'
.format(directory))
return directory
def __init__(self,
logging_directory: str,
overwrite: bool = False,
custom_logger: Optional[Logger] = None,
with_mlflow: bool = False,
mlflow_run_id: Optional[str] = None,
logging_mode: str = 'w'
):
os.makedirs(logging_directory, exist_ok=overwrite)
self.logging_directory = logging_directory
self.with_mlflow = with_mlflow
if custom_logger is not None:
self.logger = custom_logger
self.is_custom = True
else:
self.logger = getLogger(str(uuid.uuid4()))
self.log_path = os.path.join(logging_directory, 'log.txt')
self.logger.addHandler(FileHandler(self.log_path))
self.logger.setLevel(DEBUG)
self.is_custom = False
self.metrics_path = os.path.join(logging_directory, 'metrics.txt')
self.metrics = open(self.metrics_path, mode=logging_mode)
self.params = open(os.path.join(logging_directory, 'params.txt'), mode=logging_mode)
self.inherit_existing_run = False
if self.with_mlflow:
requires_mlflow()
self.mlflow_run_id = mlflow_run_id
if mlflow_run_id is not None:
self.mlflow_run_name = None
else:
self.mlflow_run_name = logging_directory