def init_logger(logger_name=__name__, output_path=None, level=logging.INFO):
"""
Initializes a logger for console and file writing.
:param logger_name: self-explanatory.
:param output_path: directory or file path where the logs should be saved.
By default it will not store file logs.
:param level: self-explanatory.
"""
logger = logging.getLogger(logger_name)
logger.setLevel(level)
formatter = logging.Formatter("%(asctime)s [%(levelname)s]: %(message)s")
# adding console output
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
if output_path:
if is_file_path(output_path):
safe_mkfdir(output_path)
if os.path.exists(output_path):
os.remove(output_path)
else:
safe_mkdir(output_path)
# using the default name of the logger
default_file_name = "log_" + strftime("%b_%d_%H_%M_%S") + '.txt'
output_path = os.path.join(output_path, default_file_name)
file_handler = logging.FileHandler(output_path)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
def __call__(self, data_path):
try:
validate_data_paths(data_path)
except Exception as e:
raise e
safe_mkdir(self.output_folder)
for dp in listify(data_path):
for file_path in get_file_paths(dp):
file_name = os.path.basename(file_path)
output_file_path = os.path.join(self.output_folder, file_name)
if not os.path.exists(output_file_path):
self._clean_and_save_file(file_path, output_file_path)
return {"data_path": self.output_folder}
def __call__(self, parent_folder_path, suffix=None):
"""
:param parent_folder_path: the path to an existing or non-existing
parent folder that should host experiment
folders.
:param suffix: TODO!
:return: a new path where artifacts can be stored.
"""
parent_folder_path = os.path.join(os.getcwd(), parent_folder_path)
safe_mkdir(parent_folder_path)
# 1. create a new folder's name
new_folder_name = self._create_new_folder_name(parent_folder_path,
suffix)
# 2. create a new folder
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
os.mkdir(new_folder_path)
return new_folder_path
def setUp(self):
self.tmp_folder = ".tmp"
safe_mkdir(self.tmp_folder)
def safe_make_folder(self, path):
safe_mkdir(path)